opendal 0.1.6.pre.rc.1

data/core/src/services/webhdfs/core.rs

@@ -0,0 +1,398 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::header::CONTENT_LENGTH;
+use http::header::CONTENT_TYPE;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+use tokio::sync::OnceCell;
+
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct WebhdfsCore {
+    pub info: Arc<AccessorInfo>,
+    pub root: String,
+    pub endpoint: String,
+    pub user_name: Option<String>,
+    pub auth: Option<String>,
+    pub root_checker: OnceCell<()>,
+
+    pub atomic_write_dir: Option<String>,
+    pub disable_list_batch: bool,
+}
+
+impl Debug for WebhdfsCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("WebhdfsCore")
+            .field("root", &self.root)
+            .field("endpoint", &self.endpoint)
+            .finish()
+    }
+}
+
+impl WebhdfsCore {
+    pub async fn webhdfs_create_dir(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=MKDIRS&overwrite=true&noredirect=true",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::put(&url);
+
+        let req = req
+            .extension(Operation::CreateDir)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    /// create object
+    pub async fn webhdfs_create_object(
+        &self,
+        path: &str,
+        size: Option<u64>,
+        args: &OpWrite,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=CREATE&overwrite=true&noredirect=true",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::put(&url);
+
+        let req = req
+            .extension(Operation::Write)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        if status != StatusCode::CREATED && status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let resp: LocationResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        let mut req = Request::put(&resp.location);
+
+        if let Some(size) = size {
+            req = req.header(CONTENT_LENGTH, size);
+        };
+
+        if let Some(content_type) = args.content_type() {
+            req = req.header(CONTENT_TYPE, content_type);
+        };
+        let req = req
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn webhdfs_rename_object(&self, from: &str, to: &str) -> Result<Response<Buffer>> {
+        let from = build_abs_path(&self.root, from);
+        let to = build_rooted_abs_path(&self.root, to);
+
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=RENAME&destination={}",
+            self.endpoint,
+            percent_encode_path(&from),
+            percent_encode_path(&to)
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += &format!("&{auth}");
+        }
+
+        let req = Request::put(&url)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    async fn webhdfs_init_append(&self, path: &str) -> Result<String> {
+        let p = build_abs_path(&self.root, path);
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=APPEND&noredirect=true",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += &format!("&{auth}");
+        }
+
+        let req = Request::post(url)
+            .extension(Operation::Write)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        let resp = self.info.http_client().send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+                let resp: LocationResponse =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                Ok(resp.location)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    pub async fn webhdfs_append(
+        &self,
+        path: &str,
+        size: u64,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let mut url = self.webhdfs_init_append(path).await?;
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += &format!("&{auth}");
+        }
+
+        let mut req = Request::post(&url);
+
+        req = req.header(CONTENT_LENGTH, size.to_string());
+
+        let req = req
+            .extension(Operation::Write)
+            .body(body)
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    /// CONCAT will concat sources to the path
+    pub async fn webhdfs_concat(
+        &self,
+        path: &str,
+        sources: Vec<String>,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let sources = sources
+            .iter()
+            .map(|p| build_rooted_abs_path(&self.root, p))
+            .collect::<Vec<String>>()
+            .join(",");
+
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=CONCAT&sources={}",
+            self.endpoint,
+            percent_encode_path(&p),
+            percent_encode_path(&sources),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += &format!("&{auth}");
+        }
+
+        let req = Request::post(url);
+
+        let req = req
+            .extension(Operation::Write)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn webhdfs_list_status(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=LISTSTATUS",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::get(&url)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn webhdfs_list_status_batch(
+        &self,
+        path: &str,
+        start_after: &str,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=LISTSTATUS_BATCH",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if !start_after.is_empty() {
+            url += format!("&startAfter={start_after}").as_str();
+        }
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::get(&url)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+        self.info.http_client().send(req).await
+    }
+
+    fn webhdfs_open_request(&self, path: &str, range: &BytesRange) -> Result<Request<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=OPEN",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += &format!("&{auth}");
+        }
+
+        if !range.is_full() {
+            url += &format!("&offset={}", range.offset());
+            if let Some(size) = range.size() {
+                url += &format!("&length={size}")
+            }
+        }
+
+        let req = Request::get(&url)
+            .extension(Operation::Read)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        Ok(req)
+    }
+
+    pub async fn webhdfs_read_file(
+        &self,
+        path: &str,
+        range: BytesRange,
+    ) -> Result<Response<HttpBody>> {
+        let req = self.webhdfs_open_request(path, &range)?;
+        self.info.http_client().fetch(req).await
+    }
+
+    pub(super) async fn webhdfs_get_file_status(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=GETFILESTATUS",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::get(&url)
+            .extension(Operation::Stat)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn webhdfs_delete(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+        let mut url = format!(
+            "{}/webhdfs/v1/{}?op=DELETE&recursive=false",
+            self.endpoint,
+            percent_encode_path(&p),
+        );
+        if let Some(user) = &self.user_name {
+            url += format!("&user.name={user}").as_str();
+        }
+        if let Some(auth) = &self.auth {
+            url += format!("&{auth}").as_str();
+        }
+
+        let req = Request::delete(&url)
+            .extension(Operation::Delete)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct LocationResponse {
+    pub location: String,
+}

data/core/src/services/webhdfs/delete.rs

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::WebhdfsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct WebhdfsDeleter {
+    core: Arc<WebhdfsCore>,
+}
+
+impl WebhdfsDeleter {
+    pub fn new(core: Arc<WebhdfsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for WebhdfsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let resp = self.core.webhdfs_delete(&path).await?;
+
+        match resp.status() {
+            StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/webhdfs/docs.md

@@ -0,0 +1,90 @@
+There two implementations of WebHDFS REST API:
+
+- Native via HDFS Namenode and Datanode, data are transferred between nodes directly.
+- [HttpFS](https://hadoop.apache.org/docs/stable/hadoop-hdfs-httpfs/index.html) is a gateway before hdfs nodes, data are proxied.
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Differences with HDFS
+
+[Hdfs][crate::services::Hdfs] is powered by HDFS's native java client. Users need to set up the HDFS services correctly. But webhdfs can access from HTTP API and no extra setup needed.
+
+## WebHDFS Compatibility Guidelines
+
+### File Creation and Write
+
+For [File creation and write](https://hadoop.apache.org/docs/r3.1.3/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Create_and_Write_to_a_File) operations,
+OpenDAL WebHDFS is optimized for Hadoop Distributed File System (HDFS) versions 2.9 and later. 
+This involves two API calls in webhdfs, where the initial `put` call to the namenode is redirected to the datanode handling the file data.
+The optional `noredirect` flag can be set to prevent redirection. If used, the API response body contains the datanode URL, which is then utilized for the subsequent `put` call with the actual file data.
+OpenDAL automatically sets the `noredirect` flag with the first `put` call. This feature is supported starting from HDFS version 2.9.
+
+### Multi-Write Support
+
+OpenDAL WebHDFS supports multi-write operations by creating temporary files in the specified `atomic_write_dir`.
+The final concatenation of these temporary files occurs when the writer is closed.
+However, it's essential to be aware of HDFS concat restrictions for earlier versions,
+where the target file must not be empty, and its last block must be full. Due to these constraints, the concat operation might fail for HDFS 2.6.
+This issue, identified as [HDFS-6641](https://issues.apache.org/jira/browse/HDFS-6641), has been addressed in later versions of HDFS.
+
+In summary, OpenDAL WebHDFS is designed for optimal compatibility with HDFS, specifically versions 2.9 and later.
+
+
+
+## Configurations
+
+- `root`: The root path of the WebHDFS service.
+- `endpoint`: The endpoint of the WebHDFS service.
+- `delegation`: The delegation token for WebHDFS.
+- `atomic_write_dir`: The tmp write dir of multi write for WebHDFS.Needs to be configured for multi write support.
+
+Refer to [`Builder`]'s public API docs for more information.
+
+## Examples
+
+### Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Webhdfs;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Webhdfs::default()
+        // set the root for WebHDFS, all operations will happen under this root
+        //
+        // Note:
+        // if the root exists, the builder will automatically create the
+        // root directory for you
+        // if the root exists and is a directory, the builder will continue working
+        // if the root exists and is a file, the builder will fail on building backend
+        .root("/path/to/dir")
+        // set the endpoint of webhdfs namenode, controlled by dfs.namenode.http-address
+        // default is http://127.0.0.1:9870
+        .endpoint("http://127.0.0.1:9870")
+        // set the delegation_token for builder
+        .delegation("delegation_token")
+        // set atomic_write_dir for builder
+        .atomic_write_dir(".opendal_tmp/");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```

data/core/src/services/webhdfs/error.rs

@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::response::Parts;
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+struct WebHdfsErrorWrapper {
+    pub remote_exception: WebHdfsError,
+}
+
+/// WebHdfsError is the error message returned by WebHdfs service
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+#[allow(dead_code)]
+struct WebHdfsError {
+    exception: String,
+    message: String,
+    java_class_name: String,
+}
+
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+    let s = String::from_utf8_lossy(&bs);
+    parse_error_msg(parts, &s)
+}
+
+pub(super) fn parse_error_msg(parts: Parts, body: &str) -> Error {
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        // passing invalid arguments will return BAD_REQUEST
+        // should be un-retryable
+        StatusCode::BAD_REQUEST => (ErrorKind::Unexpected, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match serde_json::from_str::<WebHdfsErrorWrapper>(body) {
+        Ok(wh_error) => format!("{:?}", wh_error.remote_exception),
+        Err(_) => body.to_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Buf;
+    use serde_json::from_reader;
+
+    use super::*;
+
+    /// Error response example from https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html#Error%20Responses
+    #[tokio::test]
+    async fn test_parse_error() -> Result<()> {
+        let ill_args = bytes::Bytes::from(
+            r#"
+{
+  "RemoteException":
+  {
+    "exception"    : "IllegalArgumentException",
+    "javaClassName": "java.lang.IllegalArgumentException",
+    "message"      : "Invalid value for webhdfs parameter \"permission\": ..."
+  }
+}
+    "#,
+        );
+        let body = Buffer::from(ill_args.clone());
+        let resp = Response::builder()
+            .status(StatusCode::BAD_REQUEST)
+            .body(body)
+            .unwrap();
+
+        let err = parse_error(resp);
+        assert_eq!(err.kind(), ErrorKind::Unexpected);
+        assert!(!err.is_temporary());
+
+        let err_msg: WebHdfsError = from_reader::<_, WebHdfsErrorWrapper>(ill_args.reader())
+            .expect("must success")
+            .remote_exception;
+        assert_eq!(err_msg.exception, "IllegalArgumentException");
+        assert_eq!(
+            err_msg.java_class_name,
+            "java.lang.IllegalArgumentException"
+        );
+        assert_eq!(
+            err_msg.message,
+            "Invalid value for webhdfs parameter \"permission\": ..."
+        );
+
+        Ok(())
+    }
+}