opendal 0.1.6.pre.rc.1

data/core/src/services/fs/error.rs

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+
+/// Parse error response into Error.
+pub(super) fn parse_error(e: std::io::Error) -> Error {
+    match e.kind() {
+        std::io::ErrorKind::AlreadyExists => Error::new(
+            ErrorKind::ConditionNotMatch,
+            "The file already exists in the filesystem",
+        )
+        .set_source(e),
+        _ => new_std_io_error(e),
+    }
+}

data/core/src/services/fs/lister.rs

@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::Path;
+use std::path::PathBuf;
+
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Metadata;
+use crate::Result;
+
+pub struct FsLister<P> {
+    root: PathBuf,
+
+    current_path: Option<String>,
+
+    rd: P,
+}
+
+impl<P> FsLister<P> {
+    pub fn new(root: &Path, path: &str, rd: P) -> Self {
+        Self {
+            root: root.to_owned(),
+            current_path: Some(path.to_string()),
+            rd,
+        }
+    }
+}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FsLister.
+unsafe impl<P> Sync for FsLister<P> {}
+
+impl oio::List for FsLister<tokio::fs::ReadDir> {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        // since list should return path itself, we return it first
+        if let Some(path) = self.current_path.take() {
+            let e = oio::Entry::new(path.as_str(), Metadata::new(EntryMode::DIR));
+            return Ok(Some(e));
+        }
+
+        let Some(de) = self.rd.next_entry().await.map_err(new_std_io_error)? else {
+            return Ok(None);
+        };
+
+        let entry_path = de.path();
+        let rel_path = normalize_path(
+            &entry_path
+                .strip_prefix(&self.root)
+                .expect("cannot fail because the prefix is iterated")
+                .to_string_lossy()
+                .replace('\\', "/"),
+        );
+
+        let ft = de.file_type().await.map_err(new_std_io_error)?;
+        let entry = if ft.is_dir() {
+            // Make sure we are returning the correct path.
+            oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
+        } else if ft.is_file() {
+            oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
+        } else {
+            oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
+        };
+        Ok(Some(entry))
+    }
+}

data/core/src/services/fs/mod.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for fs service.
+#[cfg(feature = "services-fs")]
+pub const FS_SCHEME: &str = "fs";
+#[cfg(feature = "services-fs")]
+mod core;
+#[cfg(feature = "services-fs")]
+mod delete;
+#[cfg(feature = "services-fs")]
+mod error;
+#[cfg(feature = "services-fs")]
+mod lister;
+#[cfg(feature = "services-fs")]
+mod reader;
+#[cfg(feature = "services-fs")]
+mod writer;
+
+#[cfg(feature = "services-fs")]
+mod backend;
+#[cfg(feature = "services-fs")]
+pub use backend::FsBuilder as Fs;
+
+mod config;
+pub use config::FsConfig;

data/core/src/services/fs/reader.rs

@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use tokio::io::AsyncReadExt;
+use tokio::io::ReadBuf;
+
+use super::core::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct FsReader<F> {
+    core: Arc<FsCore>,
+    f: F,
+    read: usize,
+    size: usize,
+    buf_size: usize,
+}
+
+impl<F> FsReader<F> {
+    pub fn new(core: Arc<FsCore>, f: F, size: usize) -> Self {
+        Self {
+            core,
+            f,
+            read: 0,
+            size,
+            // Use 2 MiB as default value.
+            buf_size: 2 * 1024 * 1024,
+        }
+    }
+}
+
+impl oio::Read for FsReader<tokio::fs::File> {
+    async fn read(&mut self) -> Result<Buffer> {
+        if self.read >= self.size {
+            return Ok(Buffer::new());
+        }
+
+        let mut bs = self.core.buf_pool.get();
+        bs.reserve(self.buf_size);
+
+        let size = (self.size - self.read).min(self.buf_size);
+        let buf = &mut bs.spare_capacity_mut()[..size];
+        let mut read_buf: ReadBuf = ReadBuf::uninit(buf);
+
+        // SAFETY: Read at most `limit` bytes into `read_buf`.
+        unsafe {
+            read_buf.assume_init(size);
+        }
+
+        let n = self
+            .f
+            .read_buf(&mut read_buf)
+            .await
+            .map_err(new_std_io_error)?;
+        self.read += n;
+
+        // Safety: We make sure that bs contains `n` more bytes.
+        let filled = read_buf.filled().len();
+        unsafe { bs.set_len(filled) }
+
+        let frozen = bs.split().freeze();
+        // Return the buffer to the pool.
+        self.core.buf_pool.put(bs);
+
+        Ok(Buffer::from(frozen))
+    }
+}

data/core/src/services/fs/writer.rs

@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use bytes::Buf;
+use tokio::io::AsyncWriteExt;
+
+use crate::raw::*;
+use crate::services::fs::core::FsCore;
+use crate::*;
+
+pub type FsWriters = TwoWays<FsWriter, oio::PositionWriter<FsWriter>>;
+
+pub struct FsWriter {
+    target_path: PathBuf,
+    /// The temp_path is used to specify whether we should move to target_path after the file has been closed.
+    temp_path: Option<PathBuf>,
+    f: tokio::fs::File,
+}
+
+impl FsWriter {
+    pub async fn create(core: Arc<FsCore>, path: &str, op: OpWrite) -> Result<Self> {
+        let target_path = core.ensure_write_abs_path(&core.root, path).await?;
+
+        // Quick path while atomic_write_dir is not set.
+        if core.atomic_write_dir.is_none() {
+            let target_file = core.fs_write(&target_path, &op).await?;
+
+            return Ok(Self {
+                target_path,
+                temp_path: None,
+                f: target_file,
+            });
+        }
+
+        let is_append = op.append();
+        let is_exist = tokio::fs::try_exists(&target_path)
+            .await
+            .map_err(new_std_io_error)?;
+        if op.if_not_exists() && is_exist {
+            return Err(Error::new(
+                ErrorKind::ConditionNotMatch,
+                "file already exists, doesn't match the condition if_not_exists",
+            ));
+        }
+
+        // The only case we allow write in place is the file
+        // exists and users request for append writing.
+        let (f, temp_path) = if !(is_append && is_exist) {
+            core.fs_tempfile_write(path).await?
+        } else {
+            let f = core.fs_write(&target_path, &op).await?;
+            (f, None)
+        };
+
+        Ok(Self {
+            target_path,
+            temp_path,
+            f,
+        })
+    }
+}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FsWriter.
+unsafe impl Sync for FsWriter {}
+
+impl oio::Write for FsWriter {
+    async fn write(&mut self, mut bs: Buffer) -> Result<()> {
+        while bs.has_remaining() {
+            let n = self.f.write(bs.chunk()).await.map_err(new_std_io_error)?;
+            bs.advance(n);
+        }
+
+        Ok(())
+    }
+
+    async fn close(&mut self) -> Result<Metadata> {
+        self.f.flush().await.map_err(new_std_io_error)?;
+        self.f.sync_all().await.map_err(new_std_io_error)?;
+
+        if let Some(temp_path) = &self.temp_path {
+            tokio::fs::rename(temp_path, &self.target_path)
+                .await
+                .map_err(new_std_io_error)?;
+        }
+
+        let file_meta = self.f.metadata().await.map_err(new_std_io_error)?;
+        let meta = Metadata::new(EntryMode::FILE)
+            .with_content_length(file_meta.len())
+            .with_last_modified(file_meta.modified().map_err(new_std_io_error)?.into());
+        Ok(meta)
+    }
+
+    async fn abort(&mut self) -> Result<()> {
+        if let Some(temp_path) = &self.temp_path {
+            tokio::fs::remove_file(temp_path)
+                .await
+                .map_err(new_std_io_error)
+        } else {
+            Err(Error::new(
+                ErrorKind::Unsupported,
+                "Fs doesn't support abort if atomic_write_dir is not set",
+            ))
+        }
+    }
+}
+
+impl oio::PositionWrite for FsWriter {
+    async fn write_all_at(&self, offset: u64, buf: Buffer) -> Result<()> {
+        let f = self
+            .f
+            .try_clone()
+            .await
+            .map_err(new_std_io_error)?
+            .into_std()
+            .await;
+
+        tokio::task::spawn_blocking(move || {
+            let mut buf = buf;
+            let mut offset = offset;
+            while !buf.is_empty() {
+                match write_at(&f, buf.chunk(), offset) {
+                    Ok(n) => {
+                        buf.advance(n);
+                        offset += n as u64
+                    }
+                    Err(e) => return Err(e),
+                }
+            }
+            Ok(())
+        })
+        .await
+        .map_err(new_task_join_error)?
+    }
+
+    async fn close(&self) -> Result<Metadata> {
+        let mut f = self
+            .f
+            .try_clone()
+            .await
+            .map_err(new_std_io_error)?
+            .into_std()
+            .await;
+
+        f.flush().map_err(new_std_io_error)?;
+        f.sync_all().map_err(new_std_io_error)?;
+
+        if let Some(temp_path) = &self.temp_path {
+            tokio::fs::rename(temp_path, &self.target_path)
+                .await
+                .map_err(new_std_io_error)?;
+        }
+
+        let file_meta = f.metadata().map_err(new_std_io_error)?;
+        let mode = if file_meta.is_file() {
+            EntryMode::FILE
+        } else if file_meta.is_dir() {
+            EntryMode::DIR
+        } else {
+            EntryMode::Unknown
+        };
+        let meta = Metadata::new(mode)
+            .with_content_length(file_meta.len())
+            .with_last_modified(file_meta.modified().map_err(new_std_io_error)?.into());
+        Ok(meta)
+    }
+
+    async fn abort(&self) -> Result<()> {
+        if let Some(temp_path) = &self.temp_path {
+            tokio::fs::remove_file(temp_path)
+                .await
+                .map_err(new_std_io_error)
+        } else {
+            Err(Error::new(
+                ErrorKind::Unsupported,
+                "Fs doesn't support abort if atomic_write_dir is not set",
+            ))
+        }
+    }
+}
+
+#[cfg(windows)]
+fn write_at(f: &File, buf: &[u8], offset: u64) -> Result<usize> {
+    use std::os::windows::fs::FileExt;
+    f.seek_write(buf, offset).map_err(new_std_io_error)
+}
+
+#[cfg(unix)]
+fn write_at(f: &File, buf: &[u8], offset: u64) -> Result<usize> {
+    use std::os::unix::fs::FileExt;
+    f.write_at(buf, offset).map_err(new_std_io_error)
+}