opendal 0.1.6.pre.rc.1

data/core/src/services/hdfs/lister.rs

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Metadata;
+use crate::Result;
+
+pub struct HdfsLister {
+    root: String,
+
+    rd: hdrs::Readdir,
+
+    current_path: Option<String>,
+}
+
+impl HdfsLister {
+    pub fn new(root: &str, rd: hdrs::Readdir, path: &str) -> Self {
+        Self {
+            root: root.to_string(),
+
+            rd,
+
+            current_path: Some(path.to_string()),
+        }
+    }
+}
+
+impl oio::List for HdfsLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        if let Some(path) = self.current_path.take() {
+            return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
+        }
+
+        let de = match self.rd.next() {
+            Some(de) => de,
+            None => return Ok(None),
+        };
+
+        let path = build_rel_path(&self.root, de.path());
+
+        let entry = if de.is_file() {
+            let meta = Metadata::new(EntryMode::FILE)
+                .with_content_length(de.len())
+                .with_last_modified(de.modified().into());
+            oio::Entry::new(&path, meta)
+        } else if de.is_dir() {
+            // Make sure we are returning the correct path.
+            oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
+        } else {
+            oio::Entry::new(&path, Metadata::new(EntryMode::Unknown))
+        };
+
+        Ok(Some(entry))
+    }
+}

data/core/src/services/hdfs/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for hdfs service.
+#[cfg(feature = "services-hdfs")]
+pub(super) const DEFAULT_SCHEME: &str = "hdfs";
+#[cfg(feature = "services-hdfs")]
+mod delete;
+#[cfg(feature = "services-hdfs")]
+mod lister;
+#[cfg(feature = "services-hdfs")]
+mod reader;
+#[cfg(feature = "services-hdfs")]
+mod writer;
+
+#[cfg(feature = "services-hdfs")]
+mod backend;
+#[cfg(feature = "services-hdfs")]
+pub use backend::HdfsBuilder as Hdfs;
+
+mod config;
+pub use config::HdfsConfig;

data/core/src/services/hdfs/reader.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::BytesMut;
+use futures::AsyncReadExt;
+use hdrs::AsyncFile;
+use tokio::io::ReadBuf;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct HdfsReader<F> {
+    f: F,
+    read: usize,
+    size: usize,
+    buf_size: usize,
+    buf: BytesMut,
+}
+
+impl<F> HdfsReader<F> {
+    pub fn new(f: F, size: usize) -> Self {
+        Self {
+            f,
+            read: 0,
+            size,
+            // Use 2 MiB as default value.
+            buf_size: 2 * 1024 * 1024,
+            buf: BytesMut::new(),
+        }
+    }
+}
+
+impl oio::Read for HdfsReader<AsyncFile> {
+    async fn read(&mut self) -> Result<Buffer> {
+        if self.read >= self.size {
+            return Ok(Buffer::new());
+        }
+
+        let size = (self.size - self.read).min(self.buf_size);
+        self.buf.reserve(size);
+
+        let buf = &mut self.buf.spare_capacity_mut()[..size];
+        let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
+
+        // SAFETY: Read at most `limit` bytes into `read_buf`.
+        unsafe {
+            read_buf.assume_init(size);
+        }
+
+        let n = self
+            .f
+            .read(read_buf.initialize_unfilled())
+            .await
+            .map_err(new_std_io_error)?;
+        read_buf.advance(n);
+        self.read += n;
+
+        // Safety: We make sure that bs contains `n` more bytes.
+        let filled = read_buf.filled().len();
+        unsafe { self.buf.set_len(filled) }
+
+        let frozen = self.buf.split().freeze();
+        Ok(Buffer::from(frozen))
+    }
+}

data/core/src/services/hdfs/writer.rs

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use futures::AsyncWriteExt;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct HdfsWriter<F> {
+    target_path: String,
+    tmp_path: Option<String>,
+    f: Option<F>,
+    client: Arc<hdrs::Client>,
+    target_path_exists: bool,
+    size: u64,
+}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for HdfsWriter.
+unsafe impl<F> Sync for HdfsWriter<F> {}
+
+impl<F> HdfsWriter<F> {
+    pub fn new(
+        target_path: String,
+        tmp_path: Option<String>,
+        f: F,
+        client: Arc<hdrs::Client>,
+        target_path_exists: bool,
+        initial_size: u64,
+    ) -> Self {
+        Self {
+            target_path,
+            tmp_path,
+            f: Some(f),
+            client,
+            target_path_exists,
+            size: initial_size,
+        }
+    }
+}
+
+impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
+    async fn write(&mut self, mut bs: Buffer) -> Result<()> {
+        let len = bs.len() as u64;
+        let f = self.f.as_mut().expect("HdfsWriter must be initialized");
+
+        while bs.has_remaining() {
+            let n = f.write(bs.chunk()).await.map_err(new_std_io_error)?;
+            bs.advance(n);
+
+            if n == 0 {
+                f.flush().await.map_err(new_std_io_error)?;
+            }
+        }
+
+        self.size += len;
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let f = self.f.as_mut().expect("HdfsWriter must be initialized");
+        f.close().await.map_err(new_std_io_error)?;
+
+        // TODO: we need to make rename async.
+        if let Some(tmp_path) = &self.tmp_path {
+            // we must delete the target_path, otherwise the rename_file operation will fail
+            if self.target_path_exists {
+                self.client
+                    .remove_file(&self.target_path)
+                    .map_err(new_std_io_error)?;
+            }
+            self.client
+                .rename_file(tmp_path, &self.target_path)
+                .map_err(new_std_io_error)?
+        }
+
+        Ok(Metadata::default().with_content_length(self.size))
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "HdfsWriter doesn't support abort",
+        ))
+    }
+}

data/core/src/services/hdfs_native/backend.rs

@@ -0,0 +1,340 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use hdfs_native::HdfsError;
+use hdfs_native::WriteOptions;
+use log::debug;
+
+use super::delete::HdfsNativeDeleter;
+use super::error::parse_hdfs_error;
+use super::lister::HdfsNativeLister;
+use super::reader::HdfsNativeReader;
+use super::writer::HdfsNativeWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::HdfsNativeConfig;
+use crate::*;
+/// [Hadoop Distributed File System (HDFS™)](https://hadoop.apache.org/) support.
+/// Using [Native Rust HDFS client](https://github.com/Kimahriman/hdfs-native).
+impl Configurator for HdfsNativeConfig {
+    type Builder = HdfsNativeBuilder;
+    fn into_builder(self) -> Self::Builder {
+        HdfsNativeBuilder { config: self }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct HdfsNativeBuilder {
+    config: HdfsNativeConfig,
+}
+
+impl Debug for HdfsNativeBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HdfsNativeBuilder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl HdfsNativeBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set name_node of this backend.
+    ///
+    /// Valid format including:
+    ///
+    /// - `default`: using the default setting based on hadoop config.
+    /// - `hdfs://127.0.0.1:9000`: connect to hdfs cluster.
+    pub fn name_node(mut self, name_node: &str) -> Self {
+        if !name_node.is_empty() {
+            // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
+            self.config.name_node = Some(name_node.trim_end_matches('/').to_string())
+        }
+
+        self
+    }
+
+    /// Enable append capacity of this backend.
+    ///
+    /// This should be disabled when HDFS runs in non-distributed mode.
+    pub fn enable_append(mut self, enable_append: bool) -> Self {
+        self.config.enable_append = enable_append;
+        self
+    }
+}
+
+impl Builder for HdfsNativeBuilder {
+    type Config = HdfsNativeConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let name_node = match &self.config.name_node {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "name_node is empty")
+                    .with_context("service", Scheme::HdfsNative));
+            }
+        };
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let client = hdfs_native::Client::new(name_node).map_err(parse_hdfs_error)?;
+
+        // need to check if root dir exists, create if not
+        Ok(HdfsNativeBackend {
+            root,
+            client: Arc::new(client),
+            enable_append: self.config.enable_append,
+        })
+    }
+}
+
+// #[inline]
+// fn tmp_file_of(path: &str) -> String {
+//     let name = get_basename(path);
+//     let uuid = Uuid::new_v4().to_string();
+
+//     format!("{name}.{uuid}")
+// }
+
+/// Backend for hdfs-native services.
+#[derive(Debug, Clone)]
+pub struct HdfsNativeBackend {
+    pub root: String,
+    pub client: Arc<hdfs_native::Client>,
+    enable_append: bool,
+}
+
+/// hdfs_native::Client is thread-safe.
+unsafe impl Send for HdfsNativeBackend {}
+unsafe impl Sync for HdfsNativeBackend {}
+
+impl Access for HdfsNativeBackend {
+    type Reader = HdfsNativeReader;
+    type Writer = HdfsNativeWriter;
+    type Lister = Option<HdfsNativeLister>;
+    type Deleter = oio::OneShotDeleter<HdfsNativeDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        let am = AccessorInfo::default();
+        am.set_scheme(DEFAULT_SCHEME)
+            .set_root(&self.root)
+            .set_native_capability(Capability {
+                stat: true,
+
+                read: true,
+
+                write: true,
+                write_can_append: self.enable_append,
+
+                create_dir: true,
+                delete: true,
+
+                list: true,
+
+                rename: true,
+
+                shared: true,
+
+                ..Default::default()
+            });
+
+        am.into()
+    }
+
+    async fn create_dir(&self, path: &str, _args: OpCreateDir) -> Result<RpCreateDir> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        self.client
+            .mkdirs(&p, 0o777, true)
+            .await
+            .map_err(parse_hdfs_error)?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let status: hdfs_native::client::FileStatus = self
+            .client
+            .get_file_info(&p)
+            .await
+            .map_err(parse_hdfs_error)?;
+
+        let mode = if status.isdir {
+            EntryMode::DIR
+        } else {
+            EntryMode::FILE
+        };
+
+        let mut metadata = Metadata::new(mode);
+        metadata
+            .set_last_modified(parse_datetime_from_from_timestamp_millis(
+                status.modification_time as i64,
+            )?)
+            .set_content_length(status.length as u64);
+
+        Ok(RpStat::new(metadata))
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let f = self.client.read(&p).await.map_err(parse_hdfs_error)?;
+
+        let r = HdfsNativeReader::new(
+            f,
+            args.range().offset() as _,
+            args.range().size().unwrap_or(u64::MAX) as _,
+        );
+
+        Ok((RpRead::new(), r))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let target_path = build_rooted_abs_path(&self.root, path);
+        let mut initial_size = 0;
+
+        let target_exists = match self.client.get_file_info(&target_path).await {
+            Ok(status) => {
+                initial_size = status.length as u64;
+                true
+            }
+            Err(err) => match &err {
+                HdfsError::FileNotFound(_) => false,
+                _ => return Err(parse_hdfs_error(err)),
+            },
+        };
+
+        let f = if target_exists {
+            if args.append() {
+                assert!(self.enable_append, "append is not enabled");
+                self.client
+                    .append(&target_path)
+                    .await
+                    .map_err(parse_hdfs_error)?
+            } else {
+                initial_size = 0;
+                self.client
+                    .create(&target_path, WriteOptions::default().overwrite(true))
+                    .await
+                    .map_err(parse_hdfs_error)?
+            }
+        } else {
+            initial_size = 0;
+            self.client
+                .create(&target_path, WriteOptions::default())
+                .await
+                .map_err(parse_hdfs_error)?
+        };
+
+        Ok((RpWrite::new(), HdfsNativeWriter::new(f, initial_size)))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(HdfsNativeDeleter::new(Arc::new(self.clone()))),
+        ))
+    }
+
+    async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> {
+        let p: String = build_rooted_abs_path(&self.root, path);
+
+        let isdir = match self.client.get_file_info(&p).await {
+            Ok(status) => status.isdir,
+            Err(err) => {
+                return match &err {
+                    HdfsError::FileNotFound(_) => Ok((RpList::default(), None)),
+                    _ => Err(parse_hdfs_error(err)),
+                };
+            }
+        };
+        let current_path = if isdir {
+            if !path.ends_with("/") {
+                Some(path.to_string() + "/")
+            } else {
+                Some(path.to_string())
+            }
+        } else {
+            None
+        };
+
+        Ok((
+            RpList::default(),
+            Some(HdfsNativeLister::new(
+                &self.root,
+                &self.client,
+                &p,
+                current_path,
+            )),
+        ))
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        let from_path = build_rooted_abs_path(&self.root, from);
+        let to_path = build_rooted_abs_path(&self.root, to);
+        match self.client.get_file_info(&to_path).await {
+            Ok(status) => {
+                if status.isdir {
+                    return Err(Error::new(ErrorKind::IsADirectory, "path should be a file")
+                        .with_context("input", &to_path));
+                } else {
+                    self.client
+                        .delete(&to_path, true)
+                        .await
+                        .map_err(parse_hdfs_error)?;
+                }
+            }
+            Err(err) => match &err {
+                HdfsError::FileNotFound(_) => {
+                    self.client
+                        .create(&to_path, WriteOptions::default().create_parent(true))
+                        .await
+                        .map_err(parse_hdfs_error)?;
+                }
+                _ => return Err(parse_hdfs_error(err)),
+            },
+        };
+
+        self.client
+            .rename(&from_path, &to_path, true)
+            .await
+            .map_err(parse_hdfs_error)?;
+
+        Ok(RpRename::default())
+    }
+}

data/core/src/services/hdfs_native/config.rs

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for HdfsNative services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct HdfsNativeConfig {
+    /// work dir of this backend
+    pub root: Option<String>,
+    /// name_node of this backend
+    pub name_node: Option<String>,
+    /// enable the append capacity
+    pub enable_append: bool,
+}
+
+impl Debug for HdfsNativeConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HdfsNativeConfig")
+            .field("root", &self.root)
+            .field("name_node", &self.name_node)
+            .field("enable_append", &self.enable_append)
+            .finish_non_exhaustive()
+    }
+}