opendal 0.1.6.pre.rc.1

data/core/src/services/foundationdb/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-foundationdb")]
+mod backend;
+#[cfg(feature = "services-foundationdb")]
+pub use backend::FoundationdbBuilder as Foundationdb;
+
+mod config;
+pub use config::FoundationdbConfig;

data/core/src/services/fs/backend.rs

@@ -0,0 +1,299 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use log::debug;
+
+use http::Uri;
+use percent_encoding::percent_decode_str;
+
+use super::core::*;
+use super::delete::FsDeleter;
+use super::lister::FsLister;
+use super::reader::FsReader;
+use super::writer::FsWriter;
+use super::writer::FsWriters;
+use super::FS_SCHEME;
+use crate::raw::*;
+use crate::services::FsConfig;
+use crate::*;
+impl Configurator for FsConfig {
+    type Builder = FsBuilder;
+    fn from_uri(uri: &Uri, options: &HashMap<String, String>) -> Result<Self> {
+        let mut map = options.clone();
+
+        if !map.contains_key("root") {
+            let path = percent_decode_str(uri.path()).decode_utf8_lossy();
+            if path.is_empty() || path == "/" {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "fs uri requires absolute path",
+                ));
+            }
+            if !path.starts_with('/') {
+                return Err(Error::new(
+                    ErrorKind::ConfigInvalid,
+                    "fs uri root must be absolute",
+                ));
+            }
+            map.insert("root".to_string(), path.to_string());
+        }
+
+        Self::from_iter(map)
+    }
+    fn into_builder(self) -> Self::Builder {
+        FsBuilder { config: self }
+    }
+}
+
+/// POSIX file system support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Debug)]
+pub struct FsBuilder {
+    config: FsConfig,
+}
+
+impl FsBuilder {
+    /// Set root for backend.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set temp dir for atomic write.
+    ///
+    /// # Notes
+    ///
+    /// - When append is enabled, we will not use atomic write
+    ///   to avoid data loss and performance issue.
+    pub fn atomic_write_dir(mut self, dir: &str) -> Self {
+        if !dir.is_empty() {
+            self.config.atomic_write_dir = Some(dir.to_string());
+        }
+
+        self
+    }
+}
+
+impl Builder for FsBuilder {
+    type Config = FsConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = match self.config.root.map(PathBuf::from) {
+            Some(root) => Ok(root),
+            None => Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "root is not specified",
+            )),
+        }?;
+        debug!("backend use root {}", root.to_string_lossy());
+
+        // If root dir is not exist, we must create it.
+        if let Err(e) = std::fs::metadata(&root) {
+            if e.kind() == std::io::ErrorKind::NotFound {
+                std::fs::create_dir_all(&root).map_err(|e| {
+                    Error::new(ErrorKind::Unexpected, "create root dir failed")
+                        .with_operation("Builder::build")
+                        .with_context("root", root.to_string_lossy())
+                        .set_source(e)
+                })?;
+            }
+        }
+
+        let atomic_write_dir = self.config.atomic_write_dir.map(PathBuf::from);
+
+        // If atomic write dir is not exist, we must create it.
+        if let Some(d) = &atomic_write_dir {
+            if let Err(e) = std::fs::metadata(d) {
+                if e.kind() == std::io::ErrorKind::NotFound {
+                    std::fs::create_dir_all(d).map_err(|e| {
+                        Error::new(ErrorKind::Unexpected, "create atomic write dir failed")
+                            .with_operation("Builder::build")
+                            .with_context("atomic_write_dir", d.to_string_lossy())
+                            .set_source(e)
+                    })?;
+                }
+            }
+        }
+
+        // Canonicalize the root directory. This should work since we already know that we can
+        // get the metadata of the path.
+        let root = root.canonicalize().map_err(|e| {
+            Error::new(
+                ErrorKind::Unexpected,
+                "canonicalize of root directory failed",
+            )
+            .set_source(e)
+        })?;
+
+        // Canonicalize the atomic_write_dir directory. This should work since we already know that
+        // we can get the metadata of the path.
+        let atomic_write_dir = atomic_write_dir
+            .map(|p| {
+                p.canonicalize().map(Some).map_err(|e| {
+                    Error::new(
+                        ErrorKind::Unexpected,
+                        "canonicalize of atomic_write_dir directory failed",
+                    )
+                    .with_operation("Builder::build")
+                    .with_context("root", root.to_string_lossy())
+                    .set_source(e)
+                })
+            })
+            .unwrap_or(Ok(None))?;
+
+        Ok(FsBackend {
+            core: Arc::new(FsCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(FS_SCHEME)
+                        .set_root(&root.to_string_lossy())
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            read: true,
+
+                            write: true,
+                            write_can_empty: true,
+                            write_can_append: true,
+                            write_can_multi: true,
+                            write_with_if_not_exists: true,
+
+                            create_dir: true,
+                            delete: true,
+
+                            list: true,
+
+                            copy: true,
+                            rename: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+
+                    am.into()
+                },
+                root,
+                atomic_write_dir,
+                buf_pool: oio::PooledBuf::new(16).with_initial_capacity(256 * 1024),
+            }),
+        })
+    }
+}
+
+/// Backend is used to serve `Accessor` support for posix-like fs.
+#[derive(Debug, Clone)]
+pub struct FsBackend {
+    core: Arc<FsCore>,
+}
+
+impl Access for FsBackend {
+    type Reader = FsReader<tokio::fs::File>;
+    type Writer = FsWriters;
+    type Lister = Option<FsLister<tokio::fs::ReadDir>>;
+    type Deleter = oio::OneShotDeleter<FsDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        self.core.fs_create_dir(path).await?;
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let m = self.core.fs_stat(path).await?;
+        Ok(RpStat::new(m))
+    }
+
+    /// # Notes
+    ///
+    /// There are three ways to get the total file length:
+    ///
+    /// - call std::fs::metadata directly and then open. (400ns)
+    /// - open file first, and then use `f.metadata()` (300ns)
+    /// - open file first, and then use `seek`. (100ns)
+    ///
+    /// Benchmark could be found [here](https://gist.github.com/Xuanwo/48f9cfbc3022ea5f865388bb62e1a70f)
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let f = self.core.fs_read(path, &args).await?;
+        let r = FsReader::new(
+            self.core.clone(),
+            f,
+            args.range().size().unwrap_or(u64::MAX) as _,
+        );
+        Ok((RpRead::new(), r))
+    }
+
+    async fn write(&self, path: &str, op: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let is_append = op.append();
+        let concurrent = op.concurrent();
+
+        let writer = FsWriter::create(self.core.clone(), path, op).await?;
+
+        let writer = if is_append {
+            FsWriters::One(writer)
+        } else {
+            FsWriters::Two(oio::PositionWriter::new(
+                self.info().clone(),
+                writer,
+                concurrent,
+            ))
+        };
+
+        Ok((RpWrite::default(), writer))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(FsDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
+        match self.core.fs_list(path).await? {
+            Some(f) => {
+                let rd = FsLister::new(&self.core.root, path, f);
+                Ok((RpList::default(), Some(rd)))
+            }
+            None => Ok((RpList::default(), None)),
+        }
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        self.core.fs_copy(from, to).await?;
+        Ok(RpCopy::default())
+    }
+
+    async fn rename(&self, from: &str, to: &str, _args: OpRename) -> Result<RpRename> {
+        self.core.fs_rename(from, to).await?;
+        Ok(RpRename::default())
+    }
+}

data/core/src/services/fs/config.rs

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// config for file system
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct FsConfig {
+    /// root dir for backend
+    pub root: Option<String>,
+
+    /// tmp dir for atomic write
+    pub atomic_write_dir: Option<String>,
+}

data/core/src/services/fs/core.rs

@@ -0,0 +1,227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::SeekFrom;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use chrono::DateTime;
+
+use super::error::*;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Debug)]
+pub struct FsCore {
+    pub info: Arc<AccessorInfo>,
+    pub root: PathBuf,
+    pub atomic_write_dir: Option<PathBuf>,
+    pub buf_pool: oio::PooledBuf,
+}
+
+impl FsCore {
+    // Build write path and ensure the parent dirs created
+    pub async fn ensure_write_abs_path(&self, parent: &Path, path: &str) -> Result<PathBuf> {
+        let p = parent.join(path);
+
+        // Create dir before write path.
+        //
+        // TODO(xuanwo): There are many works to do here:
+        //   - Is it safe to create dir concurrently?
+        //   - Do we need to extract this logic as new util functions?
+        //   - Is it better to check the parent dir exists before call mkdir?
+        let parent = PathBuf::from(&p)
+            .parent()
+            .ok_or_else(|| {
+                Error::new(
+                    ErrorKind::Unexpected,
+                    "path should have parent but not, it must be malformed",
+                )
+                .with_context("input", p.to_string_lossy())
+            })?
+            .to_path_buf();
+
+        tokio::fs::create_dir_all(&parent)
+            .await
+            .map_err(new_std_io_error)?;
+
+        Ok(p)
+    }
+
+    pub async fn fs_create_dir(&self, path: &str) -> Result<()> {
+        let p = self.root.join(path.trim_end_matches('/'));
+        tokio::fs::create_dir_all(&p)
+            .await
+            .map_err(new_std_io_error)?;
+        Ok(())
+    }
+
+    pub async fn fs_stat(&self, path: &str) -> Result<Metadata> {
+        let p = self.root.join(path.trim_end_matches('/'));
+        let meta = tokio::fs::metadata(&p).await.map_err(new_std_io_error)?;
+
+        let mode = if meta.is_dir() {
+            EntryMode::DIR
+        } else if meta.is_file() {
+            EntryMode::FILE
+        } else {
+            EntryMode::Unknown
+        };
+        let m = Metadata::new(mode)
+            .with_content_length(meta.len())
+            .with_last_modified(
+                meta.modified()
+                    .map(DateTime::from)
+                    .map_err(new_std_io_error)?,
+            );
+
+        Ok(m)
+    }
+
+    pub async fn fs_read(&self, path: &str, args: &OpRead) -> Result<tokio::fs::File> {
+        let p = self.root.join(path.trim_end_matches('/'));
+
+        let mut f = tokio::fs::OpenOptions::new()
+            .read(true)
+            .open(&p)
+            .await
+            .map_err(new_std_io_error)?;
+
+        if args.range().offset() != 0 {
+            use tokio::io::AsyncSeekExt;
+            f.seek(SeekFrom::Start(args.range().offset()))
+                .await
+                .map_err(new_std_io_error)?;
+        }
+
+        Ok(f)
+    }
+
+    pub async fn fs_write(&self, path: &PathBuf, op: &OpWrite) -> Result<tokio::fs::File> {
+        let mut open_options = tokio::fs::OpenOptions::new();
+        if op.if_not_exists() {
+            open_options.create_new(true);
+        } else {
+            open_options.create(true);
+        }
+
+        open_options.write(true);
+
+        if op.append() {
+            open_options.append(true);
+        } else {
+            open_options.truncate(true);
+        }
+
+        let f = open_options.open(path).await.map_err(parse_error)?;
+
+        Ok(f)
+    }
+
+    /// This function is used to build a tempfile for writing.
+    ///
+    /// We don't care about the OpWrite since every check should be performed on target path directly.
+    pub async fn fs_tempfile_write(
+        &self,
+        path: &str,
+    ) -> Result<(tokio::fs::File, Option<PathBuf>)> {
+        let Some(atomic_write_dir) = self.atomic_write_dir.as_ref() else {
+            return Err(Error::new(ErrorKind::Unexpected, "fs didn't configure atomic_write_dir, but we're still entering the tempfile logic. This might be a bug."));
+        };
+
+        let tmp_path = self
+            .ensure_write_abs_path(atomic_write_dir, &build_tmp_path_of(path))
+            .await?;
+
+        let mut open_options = tokio::fs::OpenOptions::new();
+
+        // tempfile should always be new file.
+        open_options.create_new(true);
+        open_options.write(true);
+        open_options.truncate(true);
+
+        let f = open_options.open(&tmp_path).await.map_err(parse_error)?;
+
+        Ok((f, Some(tmp_path)))
+    }
+
+    pub async fn fs_list(&self, path: &str) -> Result<Option<tokio::fs::ReadDir>> {
+        let p = self.root.join(path.trim_end_matches('/'));
+
+        match tokio::fs::read_dir(&p).await {
+            Ok(rd) => Ok(Some(rd)),
+            Err(e) => {
+                match e.kind() {
+                    // Return empty list if the directory not found
+                    std::io::ErrorKind::NotFound => Ok(None),
+                    // TODO: enable after our MSRV has been raised to 1.83
+                    //
+                    // If the path is not a directory, return an empty list
+                    //
+                    // The path could be a file or a symbolic link in this case.
+                    // Returning a NotADirectory error to the user isn't helpful; instead,
+                    // providing an empty directory is a more user-friendly. In fact, the dir
+                    // `path/` does not exist.
+                    // std::io::ErrorKind::NotADirectory => Ok((RpList::default(), None)),
+                    _ => {
+                        // TODO: remove this after we have MSRV 1.83
+                        #[cfg(unix)]
+                        if e.raw_os_error() == Some(20) {
+                            // On unix 20: Not a directory
+                            return Ok(None);
+                        }
+                        #[cfg(windows)]
+                        if e.raw_os_error() == Some(267) {
+                            // On windows 267: DIRECTORY
+                            return Ok(None);
+                        }
+
+                        Err(new_std_io_error(e))
+                    }
+                }
+            }
+        }
+    }
+
+    pub async fn fs_copy(&self, from: &str, to: &str) -> Result<()> {
+        let from = self.root.join(from.trim_end_matches('/'));
+        // try to get the metadata of the source file to ensure it exists
+        tokio::fs::metadata(&from).await.map_err(new_std_io_error)?;
+
+        let to = self
+            .ensure_write_abs_path(&self.root, to.trim_end_matches('/'))
+            .await?;
+
+        tokio::fs::copy(from, to).await.map_err(new_std_io_error)?;
+        Ok(())
+    }
+
+    pub async fn fs_rename(&self, from: &str, to: &str) -> Result<()> {
+        let from = self.root.join(from.trim_end_matches('/'));
+        tokio::fs::metadata(&from).await.map_err(new_std_io_error)?;
+
+        let to = self
+            .ensure_write_abs_path(&self.root, to.trim_end_matches('/'))
+            .await?;
+
+        tokio::fs::rename(from, to)
+            .await
+            .map_err(new_std_io_error)?;
+        Ok(())
+    }
+}

data/core/src/services/fs/delete.rs

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct FsDeleter {
+    core: Arc<FsCore>,
+}
+
+impl FsDeleter {
+    pub fn new(core: Arc<FsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for FsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let p = self.core.root.join(path.trim_end_matches('/'));
+
+        let meta = tokio::fs::metadata(&p).await;
+
+        match meta {
+            Ok(meta) => {
+                if meta.is_dir() {
+                    tokio::fs::remove_dir(&p).await.map_err(new_std_io_error)?;
+                } else {
+                    tokio::fs::remove_file(&p).await.map_err(new_std_io_error)?;
+                }
+                Ok(())
+            }
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            Err(err) => Err(new_std_io_error(err)),
+        }
+    }
+}

data/core/src/services/fs/docs.md

@@ -0,0 +1,49 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] append
+- [x] create_dir
+- [x] delete
+- [x] copy
+- [x] rename
+- [x] list
+- [ ] ~~presign~~
+- [x] blocking
+
+## Configuration
+
+- `root`: Set the work dir for backend.
+- 
+You can refer to [`FsBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+
+```rust,no_run
+use std::sync::Arc;
+
+use anyhow::Result;
+use opendal::services::Fs;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create fs backend builder.
+    let mut builder = Fs::default()
+        // Set the root for fs, all operations will happen under this root.
+        //
+        // NOTE: the root must be absolute path.
+        .root("/tmp");
+
+    // `Accessor` provides the low level APIs, we will use `Operator` normally.
+    let op: Operator = Operator::new(builder)?.finish();
+
+    Ok(())
+}
+```