opendal 0.1.6.pre.rc.1

data/core/src/services/hdfs/backend.rs

@@ -0,0 +1,413 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::io;
+use std::io::SeekFrom;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use log::debug;
+
+use super::delete::HdfsDeleter;
+use super::lister::HdfsLister;
+use super::reader::HdfsReader;
+use super::writer::HdfsWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::HdfsConfig;
+use crate::*;
+impl Configurator for HdfsConfig {
+    type Builder = HdfsBuilder;
+    fn into_builder(self) -> Self::Builder {
+        HdfsBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct HdfsBuilder {
+    config: HdfsConfig,
+}
+
+impl Debug for HdfsBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HdfsBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl HdfsBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set name_node of this backend.
+    ///
+    /// Valid format including:
+    ///
+    /// - `default`: using the default setting based on hadoop config.
+    /// - `hdfs://127.0.0.1:9000`: connect to hdfs cluster.
+    pub fn name_node(mut self, name_node: &str) -> Self {
+        if !name_node.is_empty() {
+            self.config.name_node = Some(name_node.to_string())
+        }
+
+        self
+    }
+
+    /// Set kerberos_ticket_cache_path of this backend
+    ///
+    /// This should be configured when kerberos is enabled.
+    pub fn kerberos_ticket_cache_path(mut self, kerberos_ticket_cache_path: &str) -> Self {
+        if !kerberos_ticket_cache_path.is_empty() {
+            self.config.kerberos_ticket_cache_path = Some(kerberos_ticket_cache_path.to_string())
+        }
+        self
+    }
+
+    /// Set user of this backend
+    pub fn user(mut self, user: &str) -> Self {
+        if !user.is_empty() {
+            self.config.user = Some(user.to_string())
+        }
+        self
+    }
+
+    /// Enable append capacity of this backend.
+    ///
+    /// This should be disabled when HDFS runs in non-distributed mode.
+    pub fn enable_append(mut self, enable_append: bool) -> Self {
+        self.config.enable_append = enable_append;
+        self
+    }
+
+    /// Set temp dir for atomic write.
+    ///
+    /// # Notes
+    ///
+    /// - When append is enabled, we will not use atomic write
+    ///   to avoid data loss and performance issue.
+    pub fn atomic_write_dir(mut self, dir: &str) -> Self {
+        self.config.atomic_write_dir = if dir.is_empty() {
+            None
+        } else {
+            Some(String::from(dir))
+        };
+        self
+    }
+}
+
+impl Builder for HdfsBuilder {
+    type Config = HdfsConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let name_node = match &self.config.name_node {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "name node is empty")
+                    .with_context("service", Scheme::Hdfs))
+            }
+        };
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let mut builder = hdrs::ClientBuilder::new(name_node);
+        if let Some(ticket_cache_path) = &self.config.kerberos_ticket_cache_path {
+            builder = builder.with_kerberos_ticket_cache_path(ticket_cache_path.as_str());
+        }
+        if let Some(user) = &self.config.user {
+            builder = builder.with_user(user.as_str());
+        }
+
+        let client = builder.connect().map_err(new_std_io_error)?;
+
+        // Create root dir if not exist.
+        if let Err(e) = client.metadata(&root) {
+            if e.kind() == io::ErrorKind::NotFound {
+                debug!("root {root} is not exist, creating now");
+
+                client.create_dir(&root).map_err(new_std_io_error)?
+            }
+        }
+
+        let atomic_write_dir = self.config.atomic_write_dir;
+
+        // If atomic write dir is not exist, we must create it.
+        if let Some(d) = &atomic_write_dir {
+            if let Err(e) = client.metadata(d) {
+                if e.kind() == io::ErrorKind::NotFound {
+                    client.create_dir(d).map_err(new_std_io_error)?
+                }
+            }
+        }
+
+        Ok(HdfsBackend {
+            info: {
+                let am = AccessorInfo::default();
+                am.set_scheme(DEFAULT_SCHEME)
+                    .set_root(&root)
+                    .set_native_capability(Capability {
+                        stat: true,
+
+                        read: true,
+
+                        write: true,
+                        write_can_append: self.config.enable_append,
+
+                        create_dir: true,
+                        delete: true,
+
+                        list: true,
+
+                        rename: true,
+
+                        shared: true,
+
+                        ..Default::default()
+                    });
+
+                am.into()
+            },
+            root,
+            atomic_write_dir,
+            client: Arc::new(client),
+        })
+    }
+}
+
+/// Backend for hdfs services.
+#[derive(Debug, Clone)]
+pub struct HdfsBackend {
+    pub info: Arc<AccessorInfo>,
+    pub root: String,
+    atomic_write_dir: Option<String>,
+    pub client: Arc<hdrs::Client>,
+}
+
+/// hdrs::Client is thread-safe.
+unsafe impl Send for HdfsBackend {}
+unsafe impl Sync for HdfsBackend {}
+
+impl Access for HdfsBackend {
+    type Reader = HdfsReader<hdrs::AsyncFile>;
+    type Writer = HdfsWriter<hdrs::AsyncFile>;
+    type Lister = Option<HdfsLister>;
+    type Deleter = oio::OneShotDeleter<HdfsDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        self.client.create_dir(&p).map_err(new_std_io_error)?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let meta = self.client.metadata(&p).map_err(new_std_io_error)?;
+
+        let mode = if meta.is_dir() {
+            EntryMode::DIR
+        } else if meta.is_file() {
+            EntryMode::FILE
+        } else {
+            EntryMode::Unknown
+        };
+        let mut m = Metadata::new(mode);
+        m.set_content_length(meta.len());
+        m.set_last_modified(meta.modified().into());
+
+        Ok(RpStat::new(m))
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let client = self.client.clone();
+        let mut f = client
+            .open_file()
+            .read(true)
+            .async_open(&p)
+            .await
+            .map_err(new_std_io_error)?;
+
+        if args.range().offset() != 0 {
+            use futures::AsyncSeekExt;
+
+            f.seek(SeekFrom::Start(args.range().offset()))
+                .await
+                .map_err(new_std_io_error)?;
+        }
+
+        Ok((
+            RpRead::new(),
+            HdfsReader::new(f, args.range().size().unwrap_or(u64::MAX) as _),
+        ))
+    }
+
+    async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let target_path = build_rooted_abs_path(&self.root, path);
+        let mut initial_size = 0;
+        let target_exists = match self.client.metadata(&target_path) {
+            Ok(meta) => {
+                initial_size = meta.len();
+                true
+            }
+            Err(err) => {
+                if err.kind() != io::ErrorKind::NotFound {
+                    return Err(new_std_io_error(err));
+                }
+                false
+            }
+        };
+
+        let should_append = op.append() && target_exists;
+        let tmp_path = self.atomic_write_dir.as_ref().and_then(|atomic_write_dir| {
+            // If the target file exists, we should append to the end of it directly.
+            (!should_append).then_some(build_rooted_abs_path(
+                atomic_write_dir,
+                &build_tmp_path_of(path),
+            ))
+        });
+
+        if !target_exists {
+            let parent = get_parent(&target_path);
+            self.client.create_dir(parent).map_err(new_std_io_error)?;
+        }
+        if !should_append {
+            initial_size = 0;
+        }
+
+        let mut open_options = self.client.open_file();
+        open_options.create(true);
+        if should_append {
+            open_options.append(true);
+        } else {
+            open_options.write(true);
+        }
+
+        let f = open_options
+            .async_open(tmp_path.as_ref().unwrap_or(&target_path))
+            .await
+            .map_err(new_std_io_error)?;
+
+        Ok((
+            RpWrite::new(),
+            HdfsWriter::new(
+                target_path,
+                tmp_path,
+                f,
+                Arc::clone(&self.client),
+                target_exists,
+                initial_size,
+            ),
+        ))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(HdfsDeleter::new(Arc::new(self.clone()))),
+        ))
+    }
+
+    async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let f = match self.client.read_dir(&p) {
+            Ok(f) => f,
+            Err(e) => {
+                return if e.kind() == io::ErrorKind::NotFound {
+                    Ok((RpList::default(), None))
+                } else {
+                    Err(new_std_io_error(e))
+                }
+            }
+        };
+
+        let rd = HdfsLister::new(&self.root, f, path);
+
+        Ok((RpList::default(), Some(rd)))
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        let from_path = build_rooted_abs_path(&self.root, from);
+        self.client.metadata(&from_path).map_err(new_std_io_error)?;
+
+        let to_path = build_rooted_abs_path(&self.root, to);
+        let result = self.client.metadata(&to_path);
+        match result {
+            Err(err) => {
+                // Early return if other error happened.
+                if err.kind() != io::ErrorKind::NotFound {
+                    return Err(new_std_io_error(err));
+                }
+
+                let parent = PathBuf::from(&to_path)
+                    .parent()
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "path should have parent but not, it must be malformed",
+                        )
+                        .with_context("input", &to_path)
+                    })?
+                    .to_path_buf();
+
+                self.client
+                    .create_dir(&parent.to_string_lossy())
+                    .map_err(new_std_io_error)?;
+            }
+            Ok(metadata) => {
+                if metadata.is_file() {
+                    self.client
+                        .remove_file(&to_path)
+                        .map_err(new_std_io_error)?;
+                } else {
+                    return Err(Error::new(ErrorKind::IsADirectory, "path should be a file")
+                        .with_context("input", &to_path));
+                }
+            }
+        }
+
+        self.client
+            .rename_file(&from_path, &to_path)
+            .map_err(new_std_io_error)?;
+
+        Ok(RpRename::new())
+    }
+}

data/core/src/services/hdfs/config.rs

@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// [Hadoop Distributed File System (HDFS™)](https://hadoop.apache.org/) support.
+///
+/// Config for Hdfs services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct HdfsConfig {
+    /// work dir of this backend
+    pub root: Option<String>,
+    /// name node of this backend
+    pub name_node: Option<String>,
+    /// kerberos_ticket_cache_path of this backend
+    pub kerberos_ticket_cache_path: Option<String>,
+    /// user of this backend
+    pub user: Option<String>,
+    /// enable the append capacity
+    pub enable_append: bool,
+    /// atomic_write_dir of this backend
+    pub atomic_write_dir: Option<String>,
+}
+
+impl Debug for HdfsConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HdfsConfig")
+            .field("root", &self.root)
+            .field("name_node", &self.name_node)
+            .field(
+                "kerberos_ticket_cache_path",
+                &self.kerberos_ticket_cache_path,
+            )
+            .field("user", &self.user)
+            .field("enable_append", &self.enable_append)
+            .field("atomic_write_dir", &self.atomic_write_dir)
+            .finish_non_exhaustive()
+    }
+}

data/core/src/services/hdfs/delete.rs

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io;
+use std::sync::Arc;
+
+use super::backend::HdfsBackend;
+use crate::raw::*;
+use crate::*;
+
+pub struct HdfsDeleter {
+    core: Arc<HdfsBackend>,
+}
+
+impl HdfsDeleter {
+    pub fn new(core: Arc<HdfsBackend>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for HdfsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = build_rooted_abs_path(&self.core.root, &path);
+
+        let meta = self.core.client.metadata(&p);
+
+        if let Err(err) = meta {
+            return if err.kind() == io::ErrorKind::NotFound {
+                Ok(())
+            } else {
+                Err(new_std_io_error(err))
+            };
+        }
+
+        // Safety: Err branch has been checked, it's OK to unwrap.
+        let meta = meta.ok().unwrap();
+
+        let result = if meta.is_dir() {
+            self.core.client.remove_dir(&p)
+        } else {
+            self.core.client.remove_file(&p)
+        };
+
+        result.map_err(new_std_io_error)?;
+
+        Ok(())
+    }
+}

data/core/src/services/hdfs/docs.md

@@ -0,0 +1,140 @@
+A distributed file system that provides high-throughput access to application data.
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [x] rename
+- [x] list
+- [ ] ~~presign~~
+- [x] blocking
+- [x] append
+
+## Differences with webhdfs
+
+[Webhdfs][crate::services::Webhdfs] is powered by hdfs's RESTful HTTP API.
+
+## Features
+
+HDFS support needs to enable feature `services-hdfs`.
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- `name_node`: Set the name node for backend.
+- `kerberos_ticket_cache_path`: Set the kerberos ticket cache path for backend, this should be gotten by `klist` after `kinit`
+- `user`: Set the user for backend
+- `enable_append`: enable the append capacity. Default is false. 
+
+Refer to [`HdfsBuilder`]'s public API docs for more information.
+
+## Environment
+
+HDFS needs some environment set correctly.
+
+- `JAVA_HOME`: the path to java home, could be found via `java -XshowSettings:properties -version`
+- `HADOOP_HOME`: the path to hadoop home, opendal relays on this env to discover hadoop jars and set `CLASSPATH` automatically.
+
+Most of the time, setting `JAVA_HOME` and `HADOOP_HOME` is enough. But there are some edge cases:
+
+- If meeting errors like the following:
+
+```shell
+error while loading shared libraries: libjvm.so: cannot open shared object file: No such file or directory
+```
+
+Java's lib are not including in pkg-config find path, please set `LD_LIBRARY_PATH`:
+
+```shell
+export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
+```
+
+The path of `libjvm.so` could be different, please keep an eye on it.
+
+- If meeting errors like the following:
+
+```shell
+(unable to get stack trace for java.lang.NoClassDefFoundError exception: ExceptionUtils::getStackTrace error.)
+```
+
+`CLASSPATH` is not set correctly or your hadoop installation is incorrect.
+
+To set `CLASSPATH`:
+```shell
+export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':'):${CLASSPATH}
+```
+
+- If HDFS has High Availability (HA) enabled with multiple available NameNodes, some configuration is required:
+1. Obtain the entire HDFS config folder (usually located at HADOOP_HOME/etc/hadoop).
+2. Set the environment variable HADOOP_CONF_DIR to the path of this folder.
+```shell
+export HADOOP_CONF_DIR=<path of the config folder>
+```
+3. Append the HADOOP_CONF_DIR to the `CLASSPATH`
+```shell
+export CLASSPATH=$HADOOP_CONF_DIR:$HADOOP_CLASSPATH:$CLASSPATH
+```
+4. Use the `cluster_name` specified in the `core-site.xml` file (located in the HADOOP_CONF_DIR folder) to replace namenode:port.
+
+```ignore
+builder.name_node("hdfs://cluster_name");
+```
+
+### macOS Specific Note
+
+If you encounter an issue during the build process on macOS with an error message similar to:
+
+```shell
+ld: unknown file type in $HADOOP_HOME/lib/native/libhdfs.so.0.0.0
+clang: error: linker command failed with exit code 1 (use -v to see invocation)
+```
+This error is likely due to the fact that the official Hadoop build includes the libhdfs.so file for the x86-64 architecture, which is not compatible with aarch64 architecture required for MacOS.
+
+To resolve this issue, you can add hdrs as a dependency in your Rust application's Cargo.toml file, and enable the vendored feature:
+
+```toml
+[dependencies]
+hdrs = { version = "<version_number>", features = ["vendored"] }
+```
+Enabling the vendored feature ensures that hdrs includes the necessary libhdfs.so library built for the correct architecture.
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Hdfs;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create fs backend builder.
+    let mut builder = Hdfs::default()
+        // Set the name node for hdfs.
+        // If the string starts with a protocol type such as file://, hdfs://, or gs://, this protocol type will be used.
+        .name_node("hdfs://127.0.0.1:9000")
+        // Set the root for hdfs, all operations will happen under this root.
+        //
+        // NOTE: the root must be absolute path.
+        .root("/tmp")
+        
+        // Enable the append capacity for hdfs. 
+        // 
+        // Note: HDFS run in non-distributed mode doesn't support append.
+        .enable_append(true);
+
+    // `Accessor` provides the low level APIs, we will use `Operator` normally.
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```