opendal 0.1.6.pre.rc.1

data/core/src/services/webhdfs/lister.rs

@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::WebhdfsCore;
+use super::error::parse_error;
+use super::message::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct WebhdfsLister {
+    core: Arc<WebhdfsCore>,
+    path: String,
+}
+
+impl WebhdfsLister {
+    pub fn new(core: Arc<WebhdfsCore>, path: &str) -> Self {
+        Self {
+            core,
+            path: path.to_string(),
+        }
+    }
+}
+
+impl oio::PageList for WebhdfsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let file_status = if self.core.disable_list_batch {
+            let resp = self.core.webhdfs_list_status(&self.path).await?;
+            match resp.status() {
+                StatusCode::OK => {
+                    ctx.done = true;
+                    ctx.entries.push_back(oio::Entry::new(
+                        format!("{}/", self.path).as_str(),
+                        Metadata::new(EntryMode::DIR),
+                    ));
+
+                    let bs = resp.into_body();
+                    serde_json::from_reader::<_, FileStatusesWrapper>(bs.reader())
+                        .map_err(new_json_deserialize_error)?
+                        .file_statuses
+                        .file_status
+                }
+                StatusCode::NOT_FOUND => {
+                    ctx.done = true;
+                    return Ok(());
+                }
+                _ => return Err(parse_error(resp)),
+            }
+        } else {
+            let resp = self
+                .core
+                .webhdfs_list_status_batch(&self.path, &ctx.token)
+                .await?;
+            match resp.status() {
+                StatusCode::OK => {
+                    let bs = resp.into_body();
+                    let res: DirectoryListingWrapper =
+                        serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+                    let directory_listing = res.directory_listing;
+                    let file_statuses = directory_listing.partial_listing.file_statuses.file_status;
+
+                    if directory_listing.remaining_entries == 0 {
+                        ctx.entries.push_back(oio::Entry::new(
+                            format!("{}/", self.path).as_str(),
+                            Metadata::new(EntryMode::DIR),
+                        ));
+
+                        ctx.done = true;
+                    } else if !file_statuses.is_empty() {
+                        ctx.token
+                            .clone_from(&file_statuses.last().unwrap().path_suffix);
+                    }
+
+                    file_statuses
+                }
+                StatusCode::NOT_FOUND => {
+                    ctx.done = true;
+                    return Ok(());
+                }
+                _ => return Err(parse_error(resp)),
+            }
+        };
+
+        for status in file_status {
+            let mut path = if self.path.is_empty() {
+                status.path_suffix.to_string()
+            } else {
+                format!("{}/{}", self.path, status.path_suffix)
+            };
+
+            let meta = match status.ty {
+                FileStatusType::Directory => Metadata::new(EntryMode::DIR),
+                FileStatusType::File => Metadata::new(EntryMode::FILE)
+                    .with_content_length(status.length)
+                    .with_last_modified(parse_datetime_from_from_timestamp_millis(
+                        status.modification_time,
+                    )?),
+            };
+
+            if meta.mode().is_file() {
+                path = path.trim_end_matches('/').to_string();
+            }
+            if meta.mode().is_dir() {
+                path += "/"
+            }
+            let entry = oio::Entry::new(&path, meta);
+            ctx.entries.push_back(entry);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/webhdfs/message.rs

@@ -0,0 +1,249 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! WebHDFS response messages
+
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+pub(super) struct BooleanResp {
+    pub boolean: bool,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct FileStatusWrapper {
+    pub file_status: FileStatus,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct FileStatusesWrapper {
+    pub file_statuses: FileStatuses,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct DirectoryListingWrapper {
+    pub directory_listing: DirectoryListing,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(super) struct DirectoryListing {
+    pub partial_listing: PartialListing,
+    pub remaining_entries: u32,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct PartialListing {
+    pub file_statuses: FileStatuses,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub(super) struct FileStatuses {
+    pub file_status: Vec<FileStatus>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct FileStatus {
+    pub length: u64,
+    pub modification_time: i64,
+
+    pub path_suffix: String,
+    #[serde(rename = "type")]
+    pub ty: FileStatusType,
+}
+
+#[derive(Debug, Default, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "UPPERCASE")]
+pub enum FileStatusType {
+    Directory,
+    #[default]
+    File,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_file_status() {
+        let json = r#"
+{
+  "FileStatus":
+  {
+    "accessTime"      : 0,
+    "blockSize"       : 0,
+    "group"           : "supergroup",
+    "length"          : 0,
+    "modificationTime": 1320173277227,
+    "owner"           : "webuser",
+    "pathSuffix"      : "",
+    "permission"      : "777",
+    "replication"     : 0,
+    "type"            : "DIRECTORY"
+  }
+}
+"#;
+        let status: FileStatusWrapper = serde_json::from_str(json).expect("must success");
+        assert_eq!(status.file_status.length, 0);
+        assert_eq!(status.file_status.modification_time, 1320173277227);
+        assert_eq!(status.file_status.path_suffix, "");
+        assert_eq!(status.file_status.ty, FileStatusType::Directory);
+    }
+
+    #[tokio::test]
+    async fn test_list_empty() {
+        let json = r#"
+    {
+        "FileStatuses": {"FileStatus":[]}
+    }
+        "#;
+        let file_statuses = serde_json::from_str::<FileStatusesWrapper>(json)
+            .expect("must success")
+            .file_statuses
+            .file_status;
+        assert!(file_statuses.is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_list_status() {
+        let json = r#"
+{
+  "FileStatuses":
+  {
+    "FileStatus":
+    [
+      {
+        "accessTime"      : 1320171722771,
+        "blockSize"       : 33554432,
+        "group"           : "supergroup",
+        "length"          : 24930,
+        "modificationTime": 1320171722771,
+        "owner"           : "webuser",
+        "pathSuffix"      : "a.patch",
+        "permission"      : "644",
+        "replication"     : 1,
+        "type"            : "FILE"
+      },
+      {
+        "accessTime"      : 0,
+        "blockSize"       : 0,
+        "group"           : "supergroup",
+        "length"          : 0,
+        "modificationTime": 1320895981256,
+        "owner"           : "szetszwo",
+        "pathSuffix"      : "bar",
+        "permission"      : "711",
+        "replication"     : 0,
+        "type"            : "DIRECTORY"
+      }
+    ]
+  }
+}
+            "#;
+
+        let file_statuses = serde_json::from_str::<FileStatusesWrapper>(json)
+            .expect("must success")
+            .file_statuses
+            .file_status;
+
+        // we should check the value of FileStatusWrapper directly.
+        assert_eq!(file_statuses.len(), 2);
+        assert_eq!(file_statuses[0].length, 24930);
+        assert_eq!(file_statuses[0].modification_time, 1320171722771);
+        assert_eq!(file_statuses[0].path_suffix, "a.patch");
+        assert_eq!(file_statuses[0].ty, FileStatusType::File);
+        assert_eq!(file_statuses[1].length, 0);
+        assert_eq!(file_statuses[1].modification_time, 1320895981256);
+        assert_eq!(file_statuses[1].path_suffix, "bar");
+        assert_eq!(file_statuses[1].ty, FileStatusType::Directory);
+    }
+
+    #[tokio::test]
+    async fn test_list_status_batch() {
+        let json = r#"
+{
+    "DirectoryListing": {
+        "partialListing": {
+            "FileStatuses": {
+                "FileStatus": [
+                    {
+                        "accessTime": 0,
+                        "blockSize": 0,
+                        "childrenNum": 0,
+                        "fileId": 16387,
+                        "group": "supergroup",
+                        "length": 0,
+                        "modificationTime": 1473305882563,
+                        "owner": "andrew",
+                        "pathSuffix": "bardir",
+                        "permission": "755",
+                        "replication": 0,
+                        "storagePolicy": 0,
+                        "type": "DIRECTORY"
+                    },
+                    {
+                        "accessTime": 1473305896945,
+                        "blockSize": 1024,
+                        "childrenNum": 0,
+                        "fileId": 16388,
+                        "group": "supergroup",
+                        "length": 0,
+                        "modificationTime": 1473305896965,
+                        "owner": "andrew",
+                        "pathSuffix": "bazfile",
+                        "permission": "644",
+                        "replication": 3,
+                        "storagePolicy": 0,
+                        "type": "FILE"
+                    }
+                ]
+            }
+        },
+        "remainingEntries": 2
+    }
+}
+        "#;
+
+        let directory_listing = serde_json::from_str::<DirectoryListingWrapper>(json)
+            .expect("must success")
+            .directory_listing;
+
+        assert_eq!(directory_listing.remaining_entries, 2);
+        assert_eq!(
+            directory_listing
+                .partial_listing
+                .file_statuses
+                .file_status
+                .len(),
+            2
+        );
+        assert_eq!(
+            directory_listing.partial_listing.file_statuses.file_status[0].path_suffix,
+            "bardir"
+        );
+        assert_eq!(
+            directory_listing.partial_listing.file_statuses.file_status[1].path_suffix,
+            "bazfile"
+        );
+    }
+}

data/core/src/services/webhdfs/mod.rs

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for webhdfs service.
+#[cfg(feature = "services-webhdfs")]
+pub(super) const DEFAULT_SCHEME: &str = "webhdfs";
+#[cfg(feature = "services-webhdfs")]
+mod delete;
+#[cfg(feature = "services-webhdfs")]
+mod error;
+#[cfg(feature = "services-webhdfs")]
+mod lister;
+#[cfg(feature = "services-webhdfs")]
+mod message;
+#[cfg(feature = "services-webhdfs")]
+mod writer;
+
+#[cfg(feature = "services-webhdfs")]
+mod backend;
+#[cfg(feature = "services-webhdfs")]
+pub use backend::WebhdfsBuilder as Webhdfs;
+
+#[cfg(feature = "services-webhdfs")]
+mod core;
+
+mod config;
+pub use config::WebhdfsConfig;

data/core/src/services/webhdfs/writer.rs

@@ -0,0 +1,177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+use uuid::Uuid;
+
+use super::core::WebhdfsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::services::webhdfs::message::FileStatusWrapper;
+use crate::*;
+
+pub type WebhdfsWriters =
+    TwoWays<oio::BlockWriter<WebhdfsWriter>, oio::AppendWriter<WebhdfsWriter>>;
+
+pub struct WebhdfsWriter {
+    core: Arc<WebhdfsCore>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl WebhdfsWriter {
+    pub fn new(core: Arc<WebhdfsCore>, op: OpWrite, path: String) -> Self {
+        WebhdfsWriter { core, op, path }
+    }
+}
+
+impl oio::BlockWrite for WebhdfsWriter {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let resp = self
+            .core
+            .webhdfs_create_object(&self.path, Some(size), &self.op, body)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_block(&self, block_id: Uuid, size: u64, body: Buffer) -> Result<()> {
+        let Some(ref atomic_write_dir) = self.core.atomic_write_dir else {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "write multi is not supported when atomic is not set",
+            ));
+        };
+        let resp = self
+            .core
+            .webhdfs_create_object(
+                &format!("{atomic_write_dir}{block_id}"),
+                Some(size),
+                &self.op,
+                body,
+            )
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_block(&self, block_ids: Vec<Uuid>) -> Result<Metadata> {
+        let Some(ref atomic_write_dir) = self.core.atomic_write_dir else {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "write multi is not supported when atomic is not set",
+            ));
+        };
+        let first_block_id = format!("{}{}", atomic_write_dir, block_ids[0].clone());
+        if block_ids.len() >= 2 {
+            let sources: Vec<String> = block_ids[1..]
+                .iter()
+                .map(|s| format!("{atomic_write_dir}{s}"))
+                .collect();
+            // concat blocks
+            let resp = self.core.webhdfs_concat(&first_block_id, sources).await?;
+
+            let status = resp.status();
+            if status != StatusCode::OK {
+                return Err(parse_error(resp));
+            }
+        }
+        // delete the path file
+        let resp = self.core.webhdfs_delete(&self.path).await?;
+
+        let status = resp.status();
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        // rename concat file to path
+        let resp = self
+            .core
+            .webhdfs_rename_object(&first_block_id, &self.path)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_block(&self, block_ids: Vec<Uuid>) -> Result<()> {
+        for block_id in block_ids {
+            let resp = self.core.webhdfs_delete(&block_id.to_string()).await?;
+            match resp.status() {
+                StatusCode::OK => {}
+                _ => return Err(parse_error(resp)),
+            }
+        }
+        Ok(())
+    }
+}
+
+impl oio::AppendWrite for WebhdfsWriter {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self.core.webhdfs_get_file_status(&self.path).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+                let file_status = serde_json::from_reader::<_, FileStatusWrapper>(bs.reader())
+                    .map_err(new_json_deserialize_error)?
+                    .file_status;
+
+                Ok(file_status.length)
+            }
+            StatusCode::NOT_FOUND => {
+                let resp = self
+                    .core
+                    .webhdfs_create_object(&self.path, None, &self.op, Buffer::new())
+                    .await?;
+
+                let status = resp.status();
+                match status {
+                    StatusCode::CREATED | StatusCode::OK => Ok(0),
+                    _ => Err(parse_error(resp)),
+                }
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, _offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let resp = self.core.webhdfs_append(&self.path, size, body).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}