opendal 0.1.6.pre.rc.1

data/core/src/services/ftp/err.rs

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use suppaftp::FtpError;
+use suppaftp::Status;
+
+use crate::Error;
+use crate::ErrorKind;
+
+pub(super) fn parse_error(err: FtpError) -> Error {
+    let (kind, retryable) = match err {
+        // Allow retry for error
+        //
+        // `{ status: NotAvailable, body: "421 There are too many connections from your internet address." }`
+        FtpError::UnexpectedResponse(ref resp) if resp.status == Status::NotAvailable => {
+            (ErrorKind::Unexpected, true)
+        }
+        FtpError::UnexpectedResponse(ref resp) if resp.status == Status::FileUnavailable => {
+            (ErrorKind::NotFound, false)
+        }
+        // Allow retry bad response.
+        FtpError::BadResponse => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let mut err = Error::new(kind, "ftp error").set_source(err);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/src/services/ftp/lister.rs

@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::str;
+use std::str::FromStr;
+use std::vec::IntoIter;
+
+use suppaftp::list::File;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct FtpLister {
+    path: String,
+    file_iter: IntoIter<String>,
+}
+
+impl FtpLister {
+    pub fn new(path: &str, files: Vec<String>) -> Self {
+        Self {
+            path: path.to_string(),
+            file_iter: files.into_iter(),
+        }
+    }
+}
+
+impl oio::List for FtpLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        let de = match self.file_iter.next() {
+            Some(file_str) => File::from_str(file_str.as_str()).map_err(|e| {
+                Error::new(ErrorKind::Unexpected, "parse file from response").set_source(e)
+            })?,
+            None => return Ok(None),
+        };
+
+        let path = self.path.to_string() + de.name();
+
+        let mut meta = if de.is_file() {
+            Metadata::new(EntryMode::FILE)
+        } else if de.is_directory() {
+            Metadata::new(EntryMode::DIR)
+        } else {
+            Metadata::new(EntryMode::Unknown)
+        };
+        meta.set_content_length(de.size() as u64);
+        meta.set_last_modified(de.modified().into());
+
+        let entry = if de.is_file() {
+            oio::Entry::new(&path, meta)
+        } else if de.is_directory() {
+            oio::Entry::new(&format!("{}/", &path), meta)
+        } else {
+            oio::Entry::new(&path, meta)
+        };
+
+        Ok(Some(entry))
+    }
+}

data/core/src/services/ftp/mod.rs

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for ftp service.
+#[cfg(feature = "services-ftp")]
+pub(super) const DEFAULT_SCHEME: &str = "ftp";
+#[cfg(feature = "services-ftp")]
+mod delete;
+#[cfg(feature = "services-ftp")]
+mod err;
+#[cfg(feature = "services-ftp")]
+mod lister;
+#[cfg(feature = "services-ftp")]
+mod reader;
+#[cfg(feature = "services-ftp")]
+mod writer;
+
+#[cfg(feature = "services-ftp")]
+mod backend;
+#[cfg(feature = "services-ftp")]
+pub use backend::FtpBuilder as Ftp;
+#[cfg(feature = "services-ftp")]
+mod core;
+
+mod config;
+
+pub use config::FtpConfig;

data/core/src/services/ftp/reader.rs

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bb8::PooledConnection;
+use bytes::BytesMut;
+use futures::AsyncRead;
+use futures::AsyncReadExt;
+
+use super::core::Manager;
+use super::err::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct FtpReader {
+    /// Keep the connection alive while data stream is alive.
+    _ftp_stream: PooledConnection<'static, Manager>,
+
+    data_stream: Box<dyn AsyncRead + Sync + Send + Unpin + 'static>,
+    chunk: usize,
+    buf: BytesMut,
+}
+
+/// # Safety
+///
+/// We only have `&mut self` for FtpReader.
+unsafe impl Sync for FtpReader {}
+
+impl FtpReader {
+    pub async fn new(
+        mut ftp_stream: PooledConnection<'static, Manager>,
+        path: String,
+        args: OpRead,
+    ) -> Result<Self> {
+        let (offset, size) = (
+            args.range().offset(),
+            args.range().size().unwrap_or(u64::MAX),
+        );
+        if offset != 0 {
+            ftp_stream
+                .resume_transfer(offset as usize)
+                .await
+                .map_err(parse_error)?;
+        }
+        let ds = ftp_stream
+            .retr_as_stream(path)
+            .await
+            .map_err(parse_error)?
+            .take(size as _);
+
+        Ok(Self {
+            _ftp_stream: ftp_stream,
+
+            data_stream: Box::new(ds),
+            chunk: 1024 * 1024,
+            buf: BytesMut::new(),
+        })
+    }
+}
+
+impl oio::Read for FtpReader {
+    async fn read(&mut self) -> Result<Buffer> {
+        self.buf.resize(self.chunk, 0);
+        let n = self
+            .data_stream
+            .read(&mut self.buf)
+            .await
+            .map_err(new_std_io_error)?;
+        Ok(Buffer::from(self.buf.split_to(n).freeze()))
+    }
+}

data/core/src/services/ftp/writer.rs

@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bb8::PooledConnection;
+use bytes::Buf;
+use futures::AsyncWrite;
+use futures::AsyncWriteExt;
+
+use super::core::Manager;
+use super::err::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct FtpWriter {
+    target_path: String,
+    tmp_path: Option<String>,
+    ftp_stream: PooledConnection<'static, Manager>,
+    data_stream: Option<Box<dyn AsyncWrite + Sync + Send + Unpin + 'static>>,
+}
+
+/// # Safety
+///
+/// We only have `&mut self` for FtpWrite.
+unsafe impl Sync for FtpWriter {}
+
+/// # TODO
+///
+/// Writer is not implemented correctly.
+///
+/// After we can use data stream, we should return it directly.
+impl FtpWriter {
+    pub fn new(
+        ftp_stream: PooledConnection<'static, Manager>,
+        target_path: String,
+        tmp_path: Option<String>,
+    ) -> Self {
+        FtpWriter {
+            target_path,
+            tmp_path,
+            ftp_stream,
+            data_stream: None,
+        }
+    }
+}
+
+impl oio::Write for FtpWriter {
+    async fn write(&mut self, mut bs: Buffer) -> Result<()> {
+        let path = if let Some(tmp_path) = &self.tmp_path {
+            tmp_path
+        } else {
+            &self.target_path
+        };
+
+        if self.data_stream.is_none() {
+            self.data_stream = Some(Box::new(
+                self.ftp_stream
+                    .append_with_stream(path)
+                    .await
+                    .map_err(parse_error)?,
+            ));
+        }
+
+        while bs.has_remaining() {
+            let n = self
+                .data_stream
+                .as_mut()
+                .unwrap()
+                .write(bs.chunk())
+                .await
+                .map_err(|err| {
+                    Error::new(ErrorKind::Unexpected, "copy from ftp stream").set_source(err)
+                })?;
+            bs.advance(n);
+        }
+
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        let data_stream = self.data_stream.take();
+        if let Some(mut data_stream) = data_stream {
+            data_stream.flush().await.map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "flush data stream failed").set_source(err)
+            })?;
+
+            self.ftp_stream
+                .finalize_put_stream(data_stream)
+                .await
+                .map_err(parse_error)?;
+
+            if let Some(tmp_path) = &self.tmp_path {
+                self.ftp_stream
+                    .rename(tmp_path, &self.target_path)
+                    .await
+                    .map_err(parse_error)?;
+            }
+        }
+
+        Ok(Metadata::default())
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "FtpWriter doesn't support abort",
+        ))
+    }
+}