opendal 0.1.6.pre.rc.1

data/core/src/services/hdfs_native/delete.rs

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::backend::HdfsNativeBackend;
+use super::error::parse_hdfs_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct HdfsNativeDeleter {
+    core: Arc<HdfsNativeBackend>,
+}
+
+impl HdfsNativeDeleter {
+    pub fn new(core: Arc<HdfsNativeBackend>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for HdfsNativeDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = build_rooted_abs_path(&self.core.root, &path);
+
+        self.core
+            .client
+            .delete(&p, true)
+            .await
+            .map_err(parse_hdfs_error)?;
+
+        Ok(())
+    }
+}

data/core/src/services/hdfs_native/docs.md

@@ -0,0 +1,35 @@
+A distributed file system that provides high-throughput access to application data.
+Using [Native Rust HDFS client](https://github.com/Kimahriman/hdfs-native).
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [x] rename
+- [x] list
+- [x] blocking
+- [x] append
+
+## Differences with webhdfs
+
+[Webhdfs][crate::services::Webhdfs] is powered by hdfs's RESTful HTTP API.
+
+## Differences with hdfs
+
+[hdfs][crate::services::Hdfs] is powered by libhdfs and require the Java dependencies
+
+## Features
+
+HDFS-native support needs to enable feature `services-hdfs-native`.
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- `name_node`: Set the name node for backend.
+- `enable_append`: enable the append capacity. Default is false. 
+

data/core/src/services/hdfs_native/error.rs

@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use hdfs_native::HdfsError;
+
+use crate::*;
+
+/// Parse hdfs-native error into opendal::Error.
+pub fn parse_hdfs_error(hdfs_error: HdfsError) -> Error {
+    let (kind, retryable, msg) = match &hdfs_error {
+        HdfsError::IOError(err) => (ErrorKind::Unexpected, false, err.to_string()),
+        HdfsError::DataTransferError(msg) => (ErrorKind::Unexpected, false, msg.clone()),
+        HdfsError::ChecksumError => (
+            ErrorKind::Unexpected,
+            false,
+            "checksums didn't match".to_string(),
+        ),
+        HdfsError::UrlParseError(err) => (ErrorKind::Unexpected, false, err.to_string()),
+        HdfsError::AlreadyExists(msg) => (ErrorKind::AlreadyExists, false, msg.clone()),
+        HdfsError::OperationFailed(msg) => (ErrorKind::Unexpected, false, msg.clone()),
+        HdfsError::RPCError(msg0, msg1) => {
+            if msg0.contains("java.io.FileNotFoundException") {
+                (ErrorKind::NotFound, false, msg1.clone())
+            } else {
+                (ErrorKind::Unexpected, false, msg1.clone())
+            }
+        }
+        HdfsError::FileNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
+        HdfsError::BlocksNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
+        HdfsError::IsADirectoryError(msg) => (ErrorKind::IsADirectory, false, msg.clone()),
+        _ => (
+            ErrorKind::Unexpected,
+            false,
+            "unexpected error from hdfs".to_string(),
+        ),
+    };
+
+    let mut err = Error::new(kind, msg).set_source(hdfs_error);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/src/services/hdfs_native/lister.rs

@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use hdfs_native::client::ListStatusIterator;
+
+use crate::raw::build_rel_path;
+use crate::raw::oio;
+use crate::raw::parse_datetime_from_from_timestamp_millis;
+use crate::services::hdfs_native::error::parse_hdfs_error;
+use crate::EntryMode;
+use crate::Metadata;
+use crate::Result;
+
+pub struct HdfsNativeLister {
+    root: String,
+    iter: ListStatusIterator,
+    current_path: Option<String>,
+    iter_to_end: bool,
+}
+
+impl HdfsNativeLister {
+    pub fn new(
+        root: &str,
+        client: &hdfs_native::Client,
+        abs_path: &str,
+        current_path: Option<String>,
+    ) -> Self {
+        HdfsNativeLister {
+            root: root.to_string(),
+            iter: client.list_status_iter(abs_path, false),
+            current_path,
+            iter_to_end: false,
+        }
+    }
+}
+
+impl oio::List for HdfsNativeLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        if self.iter_to_end {
+            return Ok(None);
+        }
+
+        if let Some(path) = self.current_path.take() {
+            return Ok(Some(oio::Entry::new(&path, Metadata::new(EntryMode::DIR))));
+        }
+
+        match self.iter.next().await {
+            Some(Ok(status)) => {
+                let path = build_rel_path(&self.root, &status.path);
+
+                let entry = if status.isdir {
+                    oio::Entry::new(&format!("{path}/"), Metadata::new(EntryMode::DIR))
+                } else {
+                    let meta = Metadata::new(EntryMode::FILE)
+                        .with_content_length(status.length as u64)
+                        .with_last_modified(parse_datetime_from_from_timestamp_millis(
+                            status.modification_time as i64,
+                        )?);
+                    oio::Entry::new(&path, meta)
+                };
+
+                Ok(Some(entry))
+            }
+            Some(Err(e)) => Err(parse_hdfs_error(e)),
+            None => {
+                self.iter_to_end = true;
+                Ok(None)
+            }
+        }
+    }
+}

data/core/src/services/hdfs_native/mod.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for hdfs_native service.
+#[cfg(feature = "services-hdfs-native")]
+pub(super) const DEFAULT_SCHEME: &str = "hdfs_native";
+#[cfg(feature = "services-hdfs-native")]
+mod delete;
+#[cfg(feature = "services-hdfs-native")]
+mod error;
+#[cfg(feature = "services-hdfs-native")]
+mod lister;
+#[cfg(feature = "services-hdfs-native")]
+mod reader;
+#[cfg(feature = "services-hdfs-native")]
+mod writer;
+
+#[cfg(feature = "services-hdfs-native")]
+mod backend;
+#[cfg(feature = "services-hdfs-native")]
+pub use backend::HdfsNativeBuilder as HdfsNative;
+
+mod config;
+
+pub use config::HdfsNativeConfig;

data/core/src/services/hdfs_native/reader.rs

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Bytes;
+use futures::StreamExt;
+use hdfs_native::file::FileReader;
+use hdfs_native::HdfsError;
+
+use crate::raw::*;
+use crate::services::hdfs_native::error::parse_hdfs_error;
+use crate::*;
+
+pub struct HdfsNativeReader {
+    read: usize,
+    size: usize,
+    stream: futures::stream::BoxStream<'static, Result<Bytes, HdfsError>>,
+}
+
+unsafe impl Sync for HdfsNativeReader {}
+
+impl HdfsNativeReader {
+    pub fn new(f: FileReader, offset: usize, size: usize) -> Self {
+        let size = size.min(f.file_length() - offset);
+        HdfsNativeReader {
+            read: 0,
+            size,
+            stream: Box::pin(f.read_range_stream(offset, size)),
+        }
+    }
+}
+
+impl oio::Read for HdfsNativeReader {
+    async fn read(&mut self) -> Result<Buffer> {
+        if self.read >= self.size {
+            return Ok(Buffer::new());
+        }
+
+        if let Some(bytes) = self.stream.as_mut().next().await {
+            let bytes = bytes.map_err(parse_hdfs_error)?;
+            let buf = Buffer::from(bytes);
+            self.read += buf.len();
+
+            Ok(buf)
+        } else {
+            Ok(Buffer::new())
+        }
+    }
+}

data/core/src/services/hdfs_native/writer.rs

@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use hdfs_native::file::FileWriter;
+
+use crate::raw::*;
+use crate::services::hdfs_native::error::parse_hdfs_error;
+use crate::*;
+pub struct HdfsNativeWriter {
+    f: FileWriter,
+    size: u64,
+}
+
+impl HdfsNativeWriter {
+    pub fn new(f: FileWriter, initial_size: u64) -> Self {
+        HdfsNativeWriter {
+            f,
+            size: initial_size,
+        }
+    }
+}
+
+impl oio::Write for HdfsNativeWriter {
+    async fn write(&mut self, mut buf: Buffer) -> Result<()> {
+        let len = buf.len() as u64;
+
+        for bs in buf.by_ref() {
+            self.f.write(bs).await.map_err(parse_hdfs_error)?;
+        }
+
+        self.size += len;
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        self.f.close().await.map_err(parse_hdfs_error)?;
+
+        Ok(Metadata::default().with_content_length(self.size))
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "HdfsNativeWriter doesn't support abort",
+        ))
+    }
+}

data/core/src/services/http/backend.rs

@@ -0,0 +1,291 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+use log::debug;
+
+use super::core::HttpCore;
+use super::error::parse_error;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::HttpConfig;
+use crate::*;
+impl Configurator for HttpConfig {
+    type Builder = HttpBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        HttpBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// HTTP Read-only service support like [Nginx](https://www.nginx.com/) and [Caddy](https://caddyserver.com/).
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct HttpBuilder {
+    config: HttpConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for HttpBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("HttpBuilder");
+
+        de.field("config", &self.config).finish()
+    }
+}
+
+impl HttpBuilder {
+    /// Set endpoint for http backend.
+    ///
+    /// For example: `https://example.com`
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        self.config.endpoint = if endpoint.is_empty() {
+            None
+        } else {
+            Some(endpoint.to_string())
+        };
+
+        self
+    }
+
+    /// set username for http backend
+    ///
+    /// default: no username
+    pub fn username(mut self, username: &str) -> Self {
+        if !username.is_empty() {
+            self.config.username = Some(username.to_owned());
+        }
+        self
+    }
+
+    /// set password for http backend
+    ///
+    /// default: no password
+    pub fn password(mut self, password: &str) -> Self {
+        if !password.is_empty() {
+            self.config.password = Some(password.to_owned());
+        }
+        self
+    }
+
+    /// set bearer token for http backend
+    ///
+    /// default: no access token
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+
+    /// Set root path of http backend.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+}
+
+impl Builder for HttpBuilder {
+    type Config = HttpConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let endpoint = match &self.config.endpoint {
+            Some(v) => v,
+            None => {
+                return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                    .with_context("service", Scheme::Http))
+            }
+        };
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let mut auth = None;
+        if let Some(username) = &self.config.username {
+            auth = Some(format_authorization_by_basic(
+                username,
+                self.config.password.as_deref().unwrap_or_default(),
+            )?);
+        }
+        if let Some(token) = &self.config.token {
+            auth = Some(format_authorization_by_bearer(token)?)
+        }
+
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME)
+            .set_root(&root)
+            .set_native_capability(Capability {
+                stat: true,
+                stat_with_if_match: true,
+                stat_with_if_none_match: true,
+
+                read: true,
+
+                read_with_if_match: true,
+                read_with_if_none_match: true,
+
+                presign: auth.is_none(),
+                presign_read: auth.is_none(),
+                presign_stat: auth.is_none(),
+
+                shared: true,
+
+                ..Default::default()
+            });
+
+        // allow deprecated api here for compatibility
+        #[allow(deprecated)]
+        if let Some(client) = self.http_client {
+            info.update_http_client(|_| client);
+        }
+
+        let accessor_info = Arc::new(info);
+
+        let core = Arc::new(HttpCore {
+            info: accessor_info,
+            endpoint: endpoint.to_string(),
+            root,
+            authorization: auth,
+        });
+
+        Ok(HttpBackend { core })
+    }
+}
+
+/// Backend is used to serve `Accessor` support for http.
+#[derive(Clone)]
+pub struct HttpBackend {
+    core: Arc<HttpCore>,
+}
+
+impl Debug for HttpBackend {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HttpBackend")
+            .field("core", &self.core)
+            .finish()
+    }
+}
+
+impl Access for HttpBackend {
+    type Reader = HttpBody;
+    type Writer = ();
+    type Lister = ();
+    type Deleter = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        // Stat root always returns a DIR.
+        if path == "/" {
+            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+        }
+
+        let resp = self.core.http_head(path, &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => parse_into_metadata(path, resp.headers()).map(RpStat::new),
+            // HTTP Server like nginx could return FORBIDDEN if auto-index
+            // is not enabled, we should ignore them.
+            StatusCode::NOT_FOUND | StatusCode::FORBIDDEN if path.ends_with('/') => {
+                Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.http_get(path, args.range(), &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+        if self.core.has_authorization() {
+            return Err(Error::new(
+                ErrorKind::Unsupported,
+                "Http doesn't support presigned request on backend with authorization",
+            ));
+        }
+
+        let req = match args.operation() {
+            PresignOperation::Stat(v) => self.core.http_head_request(path, v)?,
+            PresignOperation::Read(v) => {
+                self.core.http_get_request(path, BytesRange::default(), v)?
+            }
+            _ => {
+                return Err(Error::new(
+                    ErrorKind::Unsupported,
+                    "Http doesn't support presigned write",
+                ))
+            }
+        };
+
+        let (parts, _) = req.into_parts();
+
+        Ok(RpPresign::new(PresignedRequest::new(
+            parts.method,
+            parts.uri,
+            parts.headers,
+        )))
+    }
+}