opendal 0.1.6.pre.rc.1

data/core/src/services/alluxio/config.rs

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for alluxio services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct AlluxioConfig {
+    /// root of this backend.
+    ///
+    /// All operations will happen under this root.
+    ///
+    /// default to `/` if not set.
+    pub root: Option<String>,
+    /// endpoint of this backend.
+    ///
+    /// Endpoint must be full uri, mostly like `http://127.0.0.1:39999`.
+    pub endpoint: Option<String>,
+}
+
+impl Debug for AlluxioConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("AlluxioConfig");
+
+        d.field("root", &self.root)
+            .field("endpoint", &self.endpoint);
+
+        d.finish_non_exhaustive()
+    }
+}

data/core/src/services/alluxio/core.rs

@@ -0,0 +1,367 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use serde::Serialize;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+/// Alluxio core
+#[derive(Clone)]
+pub struct AlluxioCore {
+    pub info: Arc<AccessorInfo>,
+    /// root of this backend.
+    pub root: String,
+    /// endpoint of alluxio
+    pub endpoint: String,
+}
+
+impl Debug for AlluxioCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Backend")
+            .field("root", &self.root)
+            .field("endpoint", &self.endpoint)
+            .finish_non_exhaustive()
+    }
+}
+
+impl AlluxioCore {
+    pub async fn create_dir(&self, path: &str) -> Result<()> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let r = CreateDirRequest {
+            recursive: Some(true),
+            allow_exists: Some(true),
+        };
+
+        let body = serde_json::to_vec(&r).map_err(new_json_serialize_error)?;
+        let body = bytes::Bytes::from(body);
+
+        let mut req = Request::post(format!(
+            "{}/api/v1/paths/{}/create-directory",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        req = req.header("Content-Type", "application/json");
+
+        let req = req.extension(Operation::CreateDir);
+
+        let req = req
+            .body(Buffer::from(body))
+            .map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn create_file(&self, path: &str) -> Result<u64> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let r = CreateFileRequest {
+            recursive: Some(true),
+        };
+
+        let body = serde_json::to_vec(&r).map_err(new_json_serialize_error)?;
+        let body = bytes::Bytes::from(body);
+        let mut req = Request::post(format!(
+            "{}/api/v1/paths/{}/create-file",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        req = req.header("Content-Type", "application/json");
+
+        let req = req.extension(Operation::Write);
+
+        let req = req
+            .body(Buffer::from(body))
+            .map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let steam_id: u64 =
+                    serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
+                Ok(steam_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub(super) async fn open_file(&self, path: &str) -> Result<u64> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let req = Request::post(format!(
+            "{}/api/v1/paths/{}/open-file",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        let req = req.extension(Operation::Read);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let steam_id: u64 =
+                    serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
+                Ok(steam_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub(super) async fn delete(&self, path: &str) -> Result<()> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let req = Request::post(format!(
+            "{}/api/v1/paths/{}/delete",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        let req = req.extension(Operation::Delete);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(()),
+            _ => {
+                let err = parse_error(resp);
+                if err.kind() == ErrorKind::NotFound {
+                    return Ok(());
+                }
+                Err(err)
+            }
+        }
+    }
+
+    pub(super) async fn rename(&self, path: &str, dst: &str) -> Result<()> {
+        let path = build_rooted_abs_path(&self.root, path);
+        let dst = build_rooted_abs_path(&self.root, dst);
+
+        let req = Request::post(format!(
+            "{}/api/v1/paths/{}/rename?dst={}",
+            self.endpoint,
+            percent_encode_path(&path),
+            percent_encode_path(&dst)
+        ));
+
+        let req = req.extension(Operation::Rename);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub(super) async fn get_status(&self, path: &str) -> Result<FileInfo> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let req = Request::post(format!(
+            "{}/api/v1/paths/{}/get-status",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        let req = req.extension(Operation::Stat);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let file_info: FileInfo =
+                    serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
+                Ok(file_info)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub(super) async fn list_status(&self, path: &str) -> Result<Vec<FileInfo>> {
+        let path = build_rooted_abs_path(&self.root, path);
+
+        let req = Request::post(format!(
+            "{}/api/v1/paths/{}/list-status",
+            self.endpoint,
+            percent_encode_path(&path)
+        ));
+
+        let req = req.extension(Operation::List);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let file_infos: Vec<FileInfo> =
+                    serde_json::from_reader(body.reader()).map_err(new_json_deserialize_error)?;
+                Ok(file_infos)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    /// TODO: we should implement range support correctly.
+    ///
+    /// Please refer to [alluxio-py](https://github.com/Alluxio/alluxio-py/blob/main/alluxio/const.py#L18)
+    pub async fn read(&self, stream_id: u64, _: BytesRange) -> Result<Response<HttpBody>> {
+        let req = Request::post(format!(
+            "{}/api/v1/streams/{}/read",
+            self.endpoint, stream_id,
+        ));
+
+        let req = req.extension(Operation::Read);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub(super) async fn write(&self, stream_id: u64, body: Buffer) -> Result<usize> {
+        let req = Request::post(format!(
+            "{}/api/v1/streams/{}/write",
+            self.endpoint, stream_id
+        ));
+
+        let req = req.extension(Operation::Write);
+
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let body = resp.into_body();
+                let size: usize =
+                    serde_json::from_reader(body.reader()).map_err(new_json_serialize_error)?;
+                Ok(size)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub(super) async fn close(&self, stream_id: u64) -> Result<()> {
+        let req = Request::post(format!(
+            "{}/api/v1/streams/{}/close",
+            self.endpoint, stream_id
+        ));
+
+        let req = req.extension(Operation::Write);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct CreateFileRequest {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    recursive: Option<bool>,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct CreateDirRequest {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    recursive: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    allow_exists: Option<bool>,
+}
+
+/// Metadata of alluxio object
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(super) struct FileInfo {
+    /// The path of the object
+    pub path: String,
+    /// The last modification time of the object
+    pub last_modification_time_ms: i64,
+    /// Whether the object is a folder
+    pub folder: bool,
+    /// The length of the object in bytes
+    pub length: u64,
+}
+
+impl TryFrom<FileInfo> for Metadata {
+    type Error = Error;
+
+    fn try_from(file_info: FileInfo) -> Result<Metadata> {
+        let mut metadata = if file_info.folder {
+            Metadata::new(EntryMode::DIR)
+        } else {
+            Metadata::new(EntryMode::FILE)
+        };
+        metadata
+            .set_content_length(file_info.length)
+            .set_last_modified(parse_datetime_from_from_timestamp_millis(
+                file_info.last_modification_time_ms,
+            )?);
+        Ok(metadata)
+    }
+}

data/core/src/services/alluxio/delete.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct AlluxioDeleter {
+    core: Arc<AlluxioCore>,
+}
+
+impl AlluxioDeleter {
+    pub fn new(core: Arc<AlluxioCore>) -> Self {
+        AlluxioDeleter { core }
+    }
+}
+
+impl oio::OneShotDelete for AlluxioDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        self.core.delete(&path).await
+    }
+}

data/core/src/services/alluxio/docs.md

@@ -0,0 +1,45 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [x] rename
+- [x] list
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the work directory for backend
+- `endpoint`: Customizable endpoint setting
+
+You can refer to [`AlluxioBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Alluxio;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create backend builder
+    let mut builder = Alluxio::default()
+        // set the storage bucket for OpenDAL
+        .root("/")
+        // set the endpoint for OpenDAL
+        .endpoint("http://127.0.0.1:39999");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/alluxio/error.rs

@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// the error response of alluxio
+#[derive(Default, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+struct AlluxioError {
+    status_code: String,
+    message: String,
+}
+
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let mut kind = match parts.status.as_u16() {
+        500 => ErrorKind::Unexpected,
+        _ => ErrorKind::Unexpected,
+    };
+
+    let (message, alluxio_err) = serde_json::from_reader::<_, AlluxioError>(bs.clone().reader())
+        .map(|alluxio_err| (format!("{alluxio_err:?}"), Some(alluxio_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(alluxio_err) = alluxio_err {
+        kind = match alluxio_err.status_code.as_str() {
+            "ALREADY_EXISTS" => ErrorKind::AlreadyExists,
+            "NOT_FOUND" => ErrorKind::NotFound,
+            _ => ErrorKind::Unexpected,
+        }
+    }
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use http::StatusCode;
+
+    use super::*;
+
+    /// Error response example is from https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+    #[test]
+    fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"statusCode":"ALREADY_EXISTS","message":"The resource you requested already exist"}"#,
+                ErrorKind::AlreadyExists,
+            ),
+            (
+                r#"{"statusCode":"NOT_FOUND","message":"The resource you requested does not exist"}"#,
+                ErrorKind::NotFound,
+            ),
+            (
+                r#"{"statusCode":"INTERNAL_SERVER_ERROR","message":"Internal server error"}"#,
+                ErrorKind::Unexpected,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder()
+                .status(StatusCode::INTERNAL_SERVER_ERROR)
+                .body(body)
+                .unwrap();
+
+            let err = parse_error(resp);
+
+            assert_eq!(err.kind(), res.1);
+        }
+    }
+}

data/core/src/services/alluxio/lister.rs

@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::AlluxioCore;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::ErrorKind;
+use crate::Result;
+
+pub struct AlluxioLister {
+    core: Arc<AlluxioCore>,
+
+    path: String,
+}
+
+impl AlluxioLister {
+    pub(super) fn new(core: Arc<AlluxioCore>, path: &str) -> Self {
+        AlluxioLister {
+            core,
+            path: path.to_string(),
+        }
+    }
+}
+
+impl oio::PageList for AlluxioLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let result = self.core.list_status(&self.path).await;
+
+        match result {
+            Ok(file_infos) => {
+                ctx.done = true;
+
+                for file_info in file_infos {
+                    let path: String = file_info.path.clone();
+                    let path = if file_info.folder {
+                        format!("{path}/")
+                    } else {
+                        path
+                    };
+                    ctx.entries.push_back(Entry::new(
+                        &build_rel_path(&self.core.root, &path),
+                        file_info.try_into()?,
+                    ));
+                }
+
+                Ok(())
+            }
+            Err(e) => {
+                if e.kind() == ErrorKind::NotFound {
+                    ctx.done = true;
+                    return Ok(());
+                }
+                Err(e)
+            }
+        }
+    }
+}