opendal 0.1.6.pre.rc.1

data/core/src/services/sftp/core.rs

@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use bb8::PooledConnection;
+use bb8::RunError;
+use log::debug;
+use openssh::KnownHosts;
+use openssh::SessionBuilder;
+use openssh_sftp_client::Sftp;
+use openssh_sftp_client::SftpOptions;
+use tokio::sync::OnceCell;
+
+use super::error::is_sftp_protocol_error;
+use super::error::parse_sftp_error;
+use super::error::parse_ssh_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SftpCore {
+    pub info: Arc<AccessorInfo>,
+    pub endpoint: String,
+    pub root: String,
+    pub user: Option<String>,
+    pub key: Option<String>,
+    pub known_hosts_strategy: KnownHosts,
+
+    pub client: OnceCell<bb8::Pool<Manager>>,
+}
+
+impl Debug for SftpCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("SftpCore")
+            .field("endpoint", &self.endpoint)
+            .field("root", &self.root)
+            .finish()
+    }
+}
+
+impl SftpCore {
+    pub async fn connect(&self) -> Result<PooledConnection<'static, Manager>> {
+        let client = self
+            .client
+            .get_or_try_init(|| async {
+                bb8::Pool::builder()
+                    .max_size(64)
+                    .build(Manager {
+                        endpoint: self.endpoint.clone(),
+                        root: self.root.clone(),
+                        user: self.user.clone(),
+                        key: self.key.clone(),
+                        known_hosts_strategy: self.known_hosts_strategy.clone(),
+                    })
+                    .await
+            })
+            .await?;
+
+        client.get_owned().await.map_err(|err| match err {
+            RunError::User(err) => err,
+            RunError::TimedOut => {
+                Error::new(ErrorKind::Unexpected, "connection request: timeout").set_temporary()
+            }
+        })
+    }
+}
+
+pub struct Manager {
+    endpoint: String,
+    root: String,
+    user: Option<String>,
+    key: Option<String>,
+    known_hosts_strategy: KnownHosts,
+}
+
+impl bb8::ManageConnection for Manager {
+    type Connection = Sftp;
+    type Error = Error;
+
+    async fn connect(&self) -> Result<Self::Connection, Self::Error> {
+        let mut session = SessionBuilder::default();
+
+        if let Some(user) = &self.user {
+            session.user(user.clone());
+        }
+
+        if let Some(key) = &self.key {
+            session.keyfile(key);
+        }
+
+        session.known_hosts_check(self.known_hosts_strategy.clone());
+
+        let session = session
+            .connect(&self.endpoint)
+            .await
+            .map_err(parse_ssh_error)?;
+
+        let sftp = Sftp::from_session(session, SftpOptions::default())
+            .await
+            .map_err(parse_sftp_error)?;
+
+        if !self.root.is_empty() {
+            let mut fs = sftp.fs();
+
+            let paths = Path::new(&self.root).components();
+            let mut current = PathBuf::new();
+            for p in paths {
+                current.push(p);
+                let res = fs.create_dir(p).await;
+
+                if let Err(e) = res {
+                    // ignore error if dir already exists
+                    if !is_sftp_protocol_error(&e) {
+                        return Err(parse_sftp_error(e));
+                    }
+                }
+                fs.set_cwd(&current);
+            }
+        }
+
+        debug!("sftp connection created at {}", self.root);
+        Ok(sftp)
+    }
+
+    // Check if connect valid by checking the root path.
+    async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Self::Error> {
+        let _ = conn.fs().metadata("./").await.map_err(parse_sftp_error)?;
+
+        Ok(())
+    }
+
+    /// Always allow reuse conn.
+    fn has_broken(&self, _: &mut Self::Connection) -> bool {
+        false
+    }
+}

data/core/src/services/sftp/delete.rs

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::SftpCore;
+use super::error::is_not_found;
+use super::error::parse_sftp_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SftpDeleter {
+    core: Arc<SftpCore>,
+}
+
+impl SftpDeleter {
+    pub fn new(core: Arc<SftpCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for SftpDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let client = self.core.connect().await?;
+
+        let mut fs = client.fs();
+        fs.set_cwd(&self.core.root);
+
+        let res = if path.ends_with('/') {
+            fs.remove_dir(path).await
+        } else {
+            fs.remove_file(path).await
+        };
+
+        match res {
+            Ok(()) => Ok(()),
+            Err(e) if is_not_found(&e) => Ok(()),
+            Err(e) => Err(parse_sftp_error(e)),
+        }
+    }
+}

data/core/src/services/sftp/docs.md

@@ -0,0 +1,49 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] append
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `endpoint`: Set the endpoint for connection. The format is same as `openssh`, using either `[user@]hostname` or `ssh://[user@]hostname[:port]`. A username or port that is specified in the endpoint overrides the one set in the builder (but does not change the builder).
+- `root`: Set the work directory for backend. It uses the default directory set by the remote `sftp-server` as default
+- `user`: Set the login user
+- `key`: Set the public key for login
+- `known_hosts_strategy`: Set the strategy for known hosts, default to `Strict`
+- `enable_copy`: Set whether the remote server has copy-file extension
+
+For security reasons, it doesn't support password login, you can use public key or ssh-copy-id instead.
+
+You can refer to [`SftpBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Sftp;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Sftp::default()
+        .endpoint("127.0.0.1")
+        .user("test")
+        .key("test_key");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/sftp/error.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use openssh::Error as SshError;
+use openssh_sftp_client::error::SftpErrorKind;
+use openssh_sftp_client::Error as SftpClientError;
+
+use crate::Error;
+use crate::ErrorKind;
+
+pub fn parse_sftp_error(e: SftpClientError) -> Error {
+    let kind = match &e {
+        SftpClientError::UnsupportedSftpProtocol { version: _ } => ErrorKind::Unsupported,
+        SftpClientError::SftpError(kind, _msg) => match kind {
+            SftpErrorKind::NoSuchFile => ErrorKind::NotFound,
+            SftpErrorKind::PermDenied => ErrorKind::PermissionDenied,
+            SftpErrorKind::OpUnsupported => ErrorKind::Unsupported,
+            _ => ErrorKind::Unexpected,
+        },
+        _ => ErrorKind::Unexpected,
+    };
+
+    let mut err = Error::new(kind, "sftp error").set_source(e);
+
+    // Mark error as temporary if it's unexpected.
+    if kind == ErrorKind::Unexpected {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+pub fn parse_ssh_error(e: SshError) -> Error {
+    Error::new(ErrorKind::Unexpected, "ssh error").set_source(e)
+}
+
+pub(super) fn is_not_found(e: &SftpClientError) -> bool {
+    matches!(e, SftpClientError::SftpError(SftpErrorKind::NoSuchFile, _))
+}
+
+pub(super) fn is_sftp_protocol_error(e: &SftpClientError) -> bool {
+    matches!(e, SftpClientError::SftpError(_, _))
+}

data/core/src/services/sftp/lister.rs

@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::pin::Pin;
+
+use futures::StreamExt;
+use openssh_sftp_client::fs::DirEntry;
+use openssh_sftp_client::fs::ReadDir;
+
+use super::error::parse_sftp_error;
+use crate::raw::oio;
+use crate::raw::oio::Entry;
+use crate::Result;
+
+pub struct SftpLister {
+    dir: Pin<Box<ReadDir>>,
+    prefix: String,
+}
+
+impl SftpLister {
+    pub fn new(dir: ReadDir, path: String) -> Self {
+        let prefix = if path == "/" { "".to_owned() } else { path };
+
+        SftpLister {
+            dir: Box::pin(dir),
+            prefix,
+        }
+    }
+}
+
+impl oio::List for SftpLister {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        loop {
+            let item = self
+                .dir
+                .next()
+                .await
+                .transpose()
+                .map_err(parse_sftp_error)?;
+
+            match item {
+                Some(e) => {
+                    if e.filename().to_str() == Some("..") {
+                        continue;
+                    } else if e.filename().to_str() == Some(".") {
+                        let mut path = self.prefix.as_str();
+                        if self.prefix.is_empty() {
+                            path = "/";
+                        }
+                        return Ok(Some(Entry::new(path, e.metadata().into())));
+                    } else {
+                        return Ok(Some(map_entry(self.prefix.as_str(), e)));
+                    }
+                }
+                None => return Ok(None),
+            }
+        }
+    }
+}
+
+fn map_entry(prefix: &str, value: DirEntry) -> Entry {
+    let path = format!(
+        "{}{}{}",
+        prefix,
+        value.filename().to_str().unwrap(),
+        if value.file_type().unwrap().is_dir() {
+            "/"
+        } else {
+            ""
+        }
+    );
+
+    Entry::new(path.as_str(), value.metadata().into())
+}

data/core/src/services/sftp/mod.rs

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for sftp service.
+#[cfg(feature = "services-sftp")]
+pub(super) const DEFAULT_SCHEME: &str = "sftp";
+#[cfg(feature = "services-sftp")]
+mod delete;
+#[cfg(feature = "services-sftp")]
+mod error;
+#[cfg(feature = "services-sftp")]
+mod lister;
+#[cfg(feature = "services-sftp")]
+mod reader;
+#[cfg(feature = "services-sftp")]
+mod utils;
+#[cfg(feature = "services-sftp")]
+mod writer;
+
+#[cfg(feature = "services-sftp")]
+mod backend;
+#[cfg(feature = "services-sftp")]
+pub use backend::SftpBuilder as Sftp;
+#[cfg(feature = "services-sftp")]
+mod core;
+
+mod config;
+pub use config::SftpConfig;

data/core/src/services/sftp/reader.rs

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bb8::PooledConnection;
+use bytes::BytesMut;
+use openssh_sftp_client::file::File;
+
+use super::core::Manager;
+use super::error::parse_sftp_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct SftpReader {
+    /// Keep the connection alive while data stream is alive.
+    _conn: PooledConnection<'static, Manager>,
+
+    file: File,
+    chunk: usize,
+    size: Option<usize>,
+    read: usize,
+    buf: BytesMut,
+}
+
+impl SftpReader {
+    pub fn new(conn: PooledConnection<'static, Manager>, file: File, size: Option<u64>) -> Self {
+        Self {
+            _conn: conn,
+            file,
+            size: size.map(|v| v as usize),
+            chunk: 2 * 1024 * 1024,
+            read: 0,
+            buf: BytesMut::new(),
+        }
+    }
+}
+
+impl oio::Read for SftpReader {
+    async fn read(&mut self) -> Result<Buffer> {
+        if self.read >= self.size.unwrap_or(usize::MAX) {
+            return Ok(Buffer::new());
+        }
+
+        let size = if let Some(size) = self.size {
+            (size - self.read).min(self.chunk)
+        } else {
+            self.chunk
+        };
+        self.buf.reserve(size);
+
+        let Some(bytes) = self
+            .file
+            .read(size as u32, self.buf.split_off(0))
+            .await
+            .map_err(parse_sftp_error)?
+        else {
+            return Ok(Buffer::new());
+        };
+
+        self.read += bytes.len();
+        self.buf = bytes;
+        let bs = self.buf.split();
+        Ok(Buffer::from(bs.freeze()))
+    }
+}

data/core/src/services/sftp/utils.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use openssh_sftp_client::metadata::MetaData as SftpMeta;
+
+use crate::EntryMode;
+use crate::Metadata;
+
+/// REMOVE ME: we should not implement `From<SftpMeta> for Metadata`.
+impl From<SftpMeta> for Metadata {
+    fn from(meta: SftpMeta) -> Self {
+        let mode = meta
+            .file_type()
+            .map(|filetype| {
+                if filetype.is_file() {
+                    EntryMode::FILE
+                } else if filetype.is_dir() {
+                    EntryMode::DIR
+                } else {
+                    EntryMode::Unknown
+                }
+            })
+            .unwrap_or(EntryMode::Unknown);
+
+        let mut metadata = Metadata::new(mode);
+
+        if let Some(size) = meta.len() {
+            metadata.set_content_length(size);
+        }
+
+        if let Some(modified) = meta.modified() {
+            metadata.set_last_modified(modified.as_system_time().into());
+        }
+
+        metadata
+    }
+}

data/core/src/services/sftp/writer.rs

@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::pin::Pin;
+
+use bytes::Buf;
+use openssh_sftp_client::file::File;
+use openssh_sftp_client::file::TokioCompatFile;
+use tokio::io::AsyncWriteExt;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct SftpWriter {
+    /// TODO: maybe we can use `File` directly?
+    file: Pin<Box<TokioCompatFile>>,
+}
+
+impl SftpWriter {
+    pub fn new(file: File) -> Self {
+        SftpWriter {
+            file: Box::pin(TokioCompatFile::new(file)),
+        }
+    }
+}
+
+impl oio::Write for SftpWriter {
+    async fn write(&mut self, mut bs: Buffer) -> Result<()> {
+        while bs.has_remaining() {
+            let n = self
+                .file
+                .write(bs.chunk())
+                .await
+                .map_err(new_std_io_error)?;
+            bs.advance(n);
+        }
+
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        self.file.shutdown().await.map_err(new_std_io_error)?;
+
+        Ok(Metadata::default())
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "SftpWriter doesn't support abort",
+        ))
+    }
+}