opendal 0.1.6.pre.rc.1

data/core/edge/file_write_on_full_disk/Cargo.toml

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "edge_test_file_write_on_full_disk"
+
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+futures = "0.3"
+opendal = { path = "../..", features = ["services-memory", "services-fs"] }
+rand = "0.8"
+tokio = { version = "1", features = ["full"] }

data/core/edge/file_write_on_full_disk/README.md

@@ -0,0 +1,14 @@
+# File Write on Fill Disk
+
+Reported by [The `FsBackend` only writes partial bytes and doesn't raise any errors](https://github.com/apache/opendal/issues/3052).
+
+Setup:
+
+```shell
+fallocate -l 512K disk.img 
+mkfs disk.img 
+mkdir ./td
+sudo mount -o loop td.img ./td
+chmod a+wr ./td
+```
+

data/core/edge/file_write_on_full_disk/src/main.rs

@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::env;
+
+use opendal::services::Fs;
+use opendal::Operator;
+use opendal::Result;
+use rand::prelude::*;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let builder =
+        Fs::default().root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test"));
+    let op = Operator::new(builder)?.finish();
+
+    let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024);
+    let mut bs = vec![0; size];
+    thread_rng().fill_bytes(&mut bs);
+
+    let result = op.write("/test", bs).await;
+    // Write file with size > 512KB should fail
+    assert!(
+        result.is_err(),
+        "write file on full disk should return error"
+    );
+
+    Ok(())
+}

data/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "edge_test_aws_s3_assume_role_with_web_identity"
+
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1", features = ["full"] }
+uuid = { version = "1", features = ["serde", "v4"] }

data/core/edge/s3_aws_assume_role_with_web_identity/README.md

@@ -0,0 +1,18 @@
+# S3 AWS Assume Role With Web Identity
+
+This edge test case is for AWS s3 services that authed by `Assume Role With Web Identity`.
+
+For setup, please configure bucket and OIDC correctly, for example:
+
+```yaml
+- uses: actions/github-script@v7
+  id: id-token
+  with:
+    script: return await core.getIDToken("sts.amazonaws.com")
+    result-encoding: string
+- name: Write ID token to file
+  run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
+```
+
+And configure `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`.
+

data/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::raw::tests::init_test_service;
+use opendal::Result;
+use opendal::Scheme;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let op = init_test_service()?.expect("service must be init");
+    assert_eq!(op.info().scheme(), Scheme::S3);
+
+    let result = op
+        .exists(&uuid::Uuid::new_v4().to_string())
+        .await
+        .expect("this operation should never return error");
+    assert!(!result, "the file must be not exist");
+
+    Ok(())
+}

data/core/edge/s3_read_on_wasm/.gitignore

@@ -0,0 +1,3 @@
+pkg/
+node_modules/
+dist/

data/core/edge/s3_read_on_wasm/Cargo.toml

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "edge_test_s3_read_on_wasm"
+
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+opendal = { path = "../..", default-features = false, features = [
+  "services-s3",
+] }
+wasm-bindgen = "0.2.89"
+wasm-bindgen-futures = "0.4.39"
+
+[dev-dependencies]
+wasm-bindgen-test = "0.3.41"

data/core/edge/s3_read_on_wasm/README.md

@@ -0,0 +1,42 @@
+# S3 Read on WASM
+
+This test is used to ensure opendal can run on WASM target.
+
+## Setup
+
+Start the S3 server at `http://127.0.0.1:9900`
+
+We are using the following config values, please feel free to change based on your own needs.
+
+```rust
+let mut cfg = S3::default();
+cfg.endpoint("http://127.0.0.1:9900");
+cfg.access_key_id("minioadmin");
+cfg.secret_access_key("minioadmin");
+cfg.bucket("opendal");
+cfg.region("us-east-1");
+```
+
+## Install
+
+```shell
+cargo install wasm-pack
+```
+
+## Build
+
+```shell
+wasm-pack build
+```
+
+## Test
+
+NOTE: 
+
+- You need to have Chrome installed.
+- We can't run on node.js yet.
+- We can't run on headless chrome yet.
+
+```shell
+wasm-pack test --chrome
+```

data/core/edge/s3_read_on_wasm/src/lib.rs

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::services::S3;
+use opendal::Operator;
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+pub async fn hello_world() -> String {
+    let cfg = S3::default()
+        .endpoint("http://127.0.0.1:9000")
+        .access_key_id("minioadmin")
+        .secret_access_key("minioadmin")
+        .bucket("test")
+        .region("us-east-1");
+
+    let op = Operator::new(cfg).unwrap().finish();
+    op.write(
+        "test",
+        "Hello, WASM! We are from OpenDAL at rust side!"
+            .as_bytes()
+            .to_vec(),
+    )
+    .await
+    .unwrap();
+    let bs = op.read("test").await.unwrap().to_bytes();
+    String::from_utf8_lossy(&bs).to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use wasm_bindgen_test::wasm_bindgen_test;
+    use wasm_bindgen_test::wasm_bindgen_test_configure;
+
+    use super::*;
+
+    wasm_bindgen_test_configure!(run_in_browser);
+
+    /// This test is executed in the wasm target but rustc reports `dead_code` in wrong.
+    #[allow(dead_code)]
+    #[wasm_bindgen_test]
+    async fn test_hello_world() {
+        let s = hello_world().await;
+        assert_eq!(s, "Hello, WASM! We are from OpenDAL at rust side!")
+    }
+}

data/core/edge/s3_read_on_wasm/webdriver.json

@@ -0,0 +1,15 @@
+{
+  "moz:firefoxOptions": {
+    "prefs": {
+      "media.navigator.streams.fake": true,
+      "media.navigator.permission.disabled": true
+    },
+    "args": []
+  },
+  "goog:chromeOptions": {
+    "args": [
+      "--use-fake-device-for-media-stream",
+      "--use-fake-ui-for-media-stream"
+    ]
+  }
+}

data/core/examples/README.md

@@ -0,0 +1,23 @@
+# Apache OpenDAL™ Rust Core Examples
+
+Thank you for using OpenDAL!
+
+Those examples are designed to help you to understand how to use OpenDAL Rust Core.
+
+## Setup
+
+All examples following the same setup steps:
+
+To run this example, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
+
+Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`.
+
+```dotenv
+OPENDAL_FS_ROOT=/path/to/dir
+```
+
+into
+
+```dotenv
+OPENDAL_FS_ROOT=/tmp/
+```

data/core/examples/basic/Cargo.toml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-examples-basic"
+publish = false
+
+edition.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+futures = "0.3"
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread"] }

data/core/examples/basic/README.md

@@ -0,0 +1,15 @@
+# Basic
+
+This example will show how to use opendal to operate storage service.
+
+## Setup
+
+Refer [Setup](../README.md) to setup all examples.
+
+## Run
+
+Use `OPENDAL_TEST` to control which service to run example:
+
+```shell
+OPENDAL_TEST=fs cargo run
+```

data/core/examples/basic/src/main.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use opendal::Operator;
+use opendal::Result;
+
+async fn example(op: Operator) -> Result<()> {
+    // Write data to S3.
+    op.write("test.txt", "Hello, World!").await?;
+    println!("write succeeded");
+
+    // Read data from s3.
+    let bs = op.read("test.txt").await?;
+
+    println!("read: {}", String::from_utf8(bs.to_vec()).unwrap());
+
+    // Fetch metadata of s3.
+    let meta = op.stat("test.txt").await?;
+    println!("stat: {meta:?}");
+
+    // Delete data from s3.
+    op.delete("test.txt").await?;
+    println!("delete succeeded");
+
+    Ok(())
+}
+
+// --- The following data is not part of this example ---
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    use opendal::raw::tests::init_test_service;
+    let op = init_test_service()?.expect("OPENDAL_TEST must be set");
+
+    println!("service {:?} has been initialized", op.info());
+    example(op).await
+}

data/core/examples/concurrent-upload/Cargo.toml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-examples-concurrent-upload"
+publish = false
+
+edition.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+futures = "0.3"
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread"] }

data/core/examples/concurrent-upload/README.md

@@ -0,0 +1,15 @@
+# Concurrent Upload
+
+This example will show how to perform upload concurrently to a storage service.
+
+## Setup
+
+Refer [Setup](../README.md) to setup all examples.
+
+## Run
+
+Use `OPENDAL_TEST` to control which service to run example:
+
+```shell
+OPENDAL_TEST=s3 cargo run
+```

data/core/examples/concurrent-upload/src/main.rs

@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::time::SystemTime;
+
+use opendal::Operator;
+use opendal::Result;
+
+async fn example(op: Operator) -> Result<()> {
+    let mut w = op.writer_with("test.txt").concurrent(4).await?;
+
+    let now = SystemTime::now();
+    // Write a 10MiB chunk.
+    w.write(vec![0; 10 * 1024 * 1024]).await?;
+    println!(
+        "vec![0; 10 * 1024 * 1024] uploaded at {:03}",
+        now.elapsed().unwrap().as_secs_f64()
+    );
+    // Write another 10MiB chunk.
+    w.write(vec![1; 10 * 1024 * 1024]).await?;
+    println!(
+        "vec![1; 10 * 1024 * 1024] uploaded at {:03}",
+        now.elapsed().unwrap().as_secs_f64()
+    );
+    // Finish the upload.
+    w.close().await?;
+    println!(
+        "upload finished at {:03}",
+        now.elapsed().unwrap().as_secs_f64()
+    );
+
+    let bs = op.read("test.txt").await?;
+    println!("read: {} bytes", bs.len());
+    assert_eq!(
+        bs.to_vec(),
+        vec![0; 10 * 1024 * 1024]
+            .into_iter()
+            .chain(vec![1; 10 * 1024 * 1024])
+            .collect::<Vec<_>>()
+    );
+
+    Ok(())
+}
+
+// --- The following data is not part of this example ---
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    use opendal::raw::tests::init_test_service;
+    let op = init_test_service()?.expect("OPENDAL_TEST must be set");
+
+    println!("service {:?} has been initialized", op.info());
+    example(op).await
+}

data/core/examples/multipart-upload/Cargo.toml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "opendal-examples-multipart-upload"
+publish = false
+
+edition.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+futures = "0.3"
+opendal = { path = "../..", features = ["tests"] }
+tokio = { version = "1.47.1", features = ["macros", "rt-multi-thread"] }

data/core/examples/multipart-upload/README.md

@@ -0,0 +1,15 @@
+# Multipart Upload
+
+This example will show how to perform a multipart upload to a storage service.
+
+## Setup
+
+Refer [Setup](../README.md) to setup all examples.
+
+## Run
+
+Use `OPENDAL_TEST` to control which service to run example:
+
+```shell
+OPENDAL_TEST=s3 cargo run
+```