opendal 0.1.6.pre.rc.1

data/core/src/services/cloudflare_kv/writer.rs

@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::CloudflareKvCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::CfKvMetadata;
+use crate::*;
+
+pub struct CloudflareWriter {
+    core: Arc<CloudflareKvCore>,
+    path: String,
+}
+
+impl CloudflareWriter {
+    pub fn new(core: Arc<CloudflareKvCore>, path: String) -> Self {
+        CloudflareWriter { core, path }
+    }
+}
+
+impl oio::OneShotWrite for CloudflareWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let cf_kv_metadata = CfKvMetadata {
+            etag: build_tmp_path_of(&self.path),
+            last_modified: chrono::Local::now().to_rfc3339(),
+            content_length: bs.len(),
+            is_dir: self.path.ends_with('/'),
+        };
+
+        let resp = self
+            .core
+            .set(&self.path, bs, cf_kv_metadata.clone())
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let mut metadata = Metadata::default();
+                metadata.set_etag(&cf_kv_metadata.etag);
+                metadata
+                    .set_last_modified(parse_datetime_from_rfc3339(&cf_kv_metadata.last_modified)?);
+                metadata.set_content_length(cf_kv_metadata.content_length as u64);
+
+                Ok(metadata)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/compfs/backend.rs

@@ -0,0 +1,290 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::io::Cursor;
+use std::sync::Arc;
+
+use compio::dispatcher::Dispatcher;
+use compio::fs::OpenOptions;
+
+use super::core::CompfsCore;
+use super::delete::CompfsDeleter;
+use super::lister::CompfsLister;
+use super::reader::CompfsReader;
+use super::writer::CompfsWriter;
+use super::DEFAULT_SCHEME;
+use crate::raw::oio::OneShotDeleter;
+use crate::raw::*;
+use crate::services::CompfsConfig;
+use crate::*;
+impl Configurator for CompfsConfig {
+    type Builder = CompfsBuilder;
+    fn into_builder(self) -> Self::Builder {
+        CompfsBuilder { config: self }
+    }
+}
+
+/// [`compio`]-based file system support.
+#[derive(Debug, Clone, Default)]
+pub struct CompfsBuilder {
+    config: CompfsConfig,
+}
+
+impl CompfsBuilder {
+    /// Set root for Compfs
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+}
+
+impl Builder for CompfsBuilder {
+    type Config = CompfsConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let root = match self.config.root {
+            Some(root) => Ok(root),
+            None => Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "root is not specified",
+            )),
+        }?;
+
+        // If root dir does not exist, we must create it.
+        if let Err(e) = std::fs::metadata(&root) {
+            if e.kind() == std::io::ErrorKind::NotFound {
+                std::fs::create_dir_all(&root).map_err(|e| {
+                    Error::new(ErrorKind::Unexpected, "create root dir failed")
+                        .with_operation("Builder::build")
+                        .with_context("root", root.as_str())
+                        .set_source(e)
+                })?;
+            }
+        }
+
+        let dispatcher = Dispatcher::new().map_err(|_| {
+            Error::new(
+                ErrorKind::Unexpected,
+                "failed to initiate compio dispatcher",
+            )
+        })?;
+        let core = CompfsCore {
+            info: {
+                let am = AccessorInfo::default();
+                am.set_scheme(DEFAULT_SCHEME)
+                    .set_root(&root)
+                    .set_native_capability(Capability {
+                        stat: true,
+
+                        read: true,
+
+                        write: true,
+                        write_can_empty: true,
+                        write_can_multi: true,
+                        create_dir: true,
+                        delete: true,
+
+                        list: true,
+
+                        copy: true,
+                        rename: true,
+
+                        shared: true,
+
+                        ..Default::default()
+                    });
+
+                am.into()
+            },
+            root: root.into(),
+            dispatcher,
+            buf_pool: oio::PooledBuf::new(16),
+        };
+        Ok(CompfsBackend {
+            core: Arc::new(core),
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct CompfsBackend {
+    core: Arc<CompfsCore>,
+}
+
+impl Access for CompfsBackend {
+    type Reader = CompfsReader;
+    type Writer = CompfsWriter;
+    type Lister = Option<CompfsLister>;
+    type Deleter = OneShotDeleter<CompfsDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        let path = self.core.prepare_path(path);
+
+        self.core
+            .exec(move || async move { compio::fs::create_dir_all(path).await })
+            .await?;
+
+        Ok(RpCreateDir::default())
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let path = self.core.prepare_path(path);
+
+        let meta = self
+            .core
+            .exec(move || async move { compio::fs::metadata(path).await })
+            .await?;
+        let ty = meta.file_type();
+        let mode = if ty.is_dir() {
+            EntryMode::DIR
+        } else if ty.is_file() {
+            EntryMode::FILE
+        } else {
+            EntryMode::Unknown
+        };
+        let last_mod = meta.modified().map_err(new_std_io_error)?.into();
+        let ret = Metadata::new(mode)
+            .with_last_modified(last_mod)
+            .with_content_length(meta.len());
+
+        Ok(RpStat::new(ret))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            OneShotDeleter::new(CompfsDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _: OpCopy) -> Result<RpCopy> {
+        let from = self.core.prepare_path(from);
+        let to = self.core.prepare_path(to);
+
+        self.core
+            .exec(move || async move {
+                let from = OpenOptions::new().read(true).open(from).await?;
+                if let Some(parent) = to.parent() {
+                    compio::fs::create_dir_all(parent).await?;
+                }
+                let to = OpenOptions::new()
+                    .write(true)
+                    .create(true)
+                    .truncate(true)
+                    .open(to)
+                    .await?;
+
+                let (mut from, mut to) = (Cursor::new(from), Cursor::new(to));
+                compio::io::copy(&mut from, &mut to).await?;
+
+                Ok(())
+            })
+            .await?;
+
+        Ok(RpCopy::default())
+    }
+
+    async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result<RpRename> {
+        let from = self.core.prepare_path(from);
+        let to = self.core.prepare_path(to);
+
+        self.core
+            .exec(move || async move {
+                if let Some(parent) = to.parent() {
+                    compio::fs::create_dir_all(parent).await?;
+                }
+                compio::fs::rename(from, to).await
+            })
+            .await?;
+
+        Ok(RpRename::default())
+    }
+
+    async fn read(&self, path: &str, op: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let path = self.core.prepare_path(path);
+
+        let file = self
+            .core
+            .exec(|| async move { compio::fs::OpenOptions::new().read(true).open(&path).await })
+            .await?;
+
+        let r = CompfsReader::new(self.core.clone(), file, op.range());
+        Ok((RpRead::new(), r))
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let path = self.core.prepare_path(path);
+        let append = args.append();
+        let file = self
+            .core
+            .exec(move || async move {
+                if let Some(parent) = path.parent() {
+                    compio::fs::create_dir_all(parent).await?;
+                }
+                let file = compio::fs::OpenOptions::new()
+                    .create(true)
+                    .write(true)
+                    .truncate(!append)
+                    .open(path)
+                    .await?;
+                let mut file = Cursor::new(file);
+                if append {
+                    let len = file.get_ref().metadata().await?.len();
+                    file.set_position(len);
+                }
+                Ok(file)
+            })
+            .await?;
+
+        let w = CompfsWriter::new(self.core.clone(), file);
+        Ok((RpWrite::new(), w))
+    }
+
+    async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> {
+        let path = self.core.prepare_path(path);
+
+        let read_dir = match self
+            .core
+            .exec_blocking({
+                let path = path.clone();
+                move || std::fs::read_dir(path)
+            })
+            .await?
+        {
+            Ok(rd) => rd,
+            Err(e) => {
+                return if e.kind() == std::io::ErrorKind::NotFound {
+                    Ok((RpList::default(), None))
+                } else {
+                    Err(new_std_io_error(e))
+                };
+            }
+        };
+
+        let lister = CompfsLister::new(self.core.clone(), &path, read_dir);
+        Ok((RpList::default(), Some(lister)))
+    }
+}

data/core/src/services/compfs/config.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// compio-based file system support.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct CompfsConfig {
+    /// root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub root: Option<String>,
+}

data/core/src/services/compfs/core.rs

@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use compio::buf::IoBuf;
+use compio::dispatcher::Dispatcher;
+
+use crate::raw::*;
+use crate::*;
+
+unsafe impl IoBuf for Buffer {
+    fn as_buf_ptr(&self) -> *const u8 {
+        self.current().as_ptr()
+    }
+
+    fn buf_len(&self) -> usize {
+        self.current().len()
+    }
+
+    fn buf_capacity(&self) -> usize {
+        // `Bytes` doesn't expose uninitialized capacity, so treat it as the same as `len`
+        self.current().len()
+    }
+}
+
+#[derive(Debug)]
+pub(super) struct CompfsCore {
+    pub info: Arc<AccessorInfo>,
+
+    pub root: PathBuf,
+    pub dispatcher: Dispatcher,
+    pub buf_pool: oio::PooledBuf,
+}
+
+impl CompfsCore {
+    pub fn prepare_path(&self, path: &str) -> PathBuf {
+        self.root.join(path.trim_end_matches('/'))
+    }
+
+    pub async fn exec<Fn, Fut, R>(&self, f: Fn) -> crate::Result<R>
+    where
+        Fn: FnOnce() -> Fut + Send + 'static,
+        Fut: Future<Output = std::io::Result<R>> + 'static,
+        R: Send + 'static,
+    {
+        self.dispatcher
+            .dispatch(f)
+            .map_err(|_| Error::new(ErrorKind::Unexpected, "compio spawn io task failed"))?
+            .await
+            .map_err(|_| Error::new(ErrorKind::Unexpected, "compio task cancelled"))?
+            .map_err(new_std_io_error)
+    }
+
+    pub async fn exec_blocking<Fn, R>(&self, f: Fn) -> Result<R>
+    where
+        Fn: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.dispatcher
+            .dispatch_blocking(f)
+            .map_err(|_| Error::new(ErrorKind::Unexpected, "compio spawn blocking task failed"))?
+            .await
+            .map_err(|_| Error::new(ErrorKind::Unexpected, "compio task cancelled"))
+    }
+}
+
+// TODO: impl IoVectoredBuf for Buffer
+// impl IoVectoredBuf for Buffer {
+//     fn as_dyn_bufs(&self) -> impl Iterator<Item = &dyn IoBuf> {}
+//
+//     fn owned_iter(self) -> Result<OwnedIter<impl OwnedIterator<Inner = Self>>, Self> {
+//         Ok(OwnedIter::new(BufferIter {
+//             current: self.current(),
+//             buf: self,
+//         }))
+//     }
+// }
+
+// #[derive(Debug, Clone)]
+// struct BufferIter {
+//     buf: Buffer,
+//     current: Bytes,
+// }
+
+// impl IntoInner for BufferIter {
+//     type Inner = Buffer;
+//
+//     fn into_inner(self) -> Self::Inner {
+//         self.buf
+//     }
+// }
+
+// impl OwnedIterator for BufferIter {
+//     fn next(mut self) -> Result<Self, Self::Inner> {
+//         let Some(current) = self.buf.next() else {
+//             return Err(self.buf);
+//         };
+//         self.current = current;
+//         Ok(self)
+//     }
+//
+//     fn current(&self) -> &dyn IoBuf {
+//         &self.current
+//     }
+// }
+
+#[cfg(test)]
+mod tests {
+    use bytes::Buf;
+    use bytes::Bytes;
+    use rand::thread_rng;
+    use rand::Rng;
+
+    use super::*;
+
+    fn setup_buffer() -> (Buffer, usize, Bytes) {
+        let mut rng = thread_rng();
+
+        let bs = (0..100)
+            .map(|_| {
+                let len = rng.gen_range(1..100);
+                let mut buf = vec![0; len];
+                rng.fill(&mut buf[..]);
+                Bytes::from(buf)
+            })
+            .collect::<Vec<_>>();
+
+        let total_size = bs.iter().map(|b| b.len()).sum::<usize>();
+        let total_content = bs.iter().flatten().copied().collect::<Bytes>();
+        let buf = Buffer::from(bs);
+
+        (buf, total_size, total_content)
+    }
+
+    #[test]
+    fn test_io_buf() {
+        let (buf, _len, _bytes) = setup_buffer();
+        let slice = IoBuf::as_slice(&buf);
+
+        assert_eq!(slice, buf.current().chunk())
+    }
+}

data/core/src/services/compfs/delete.rs

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use super::core::*;
+use crate::raw::*;
+use crate::*;
+
+pub struct CompfsDeleter {
+    core: Arc<CompfsCore>,
+}
+
+impl CompfsDeleter {
+    pub fn new(core: Arc<CompfsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for CompfsDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let res = if path.ends_with('/') {
+            let path = self.core.prepare_path(&path);
+            self.core
+                .exec(move || async move { compio::fs::remove_dir(path).await })
+                .await
+        } else {
+            let path = self.core.prepare_path(&path);
+            self.core
+                .exec(move || async move { compio::fs::remove_file(path).await })
+                .await
+        };
+        match res {
+            Ok(()) => Ok(()),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+}

data/core/src/services/compfs/lister.rs

@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fs::ReadDir;
+use std::path::Path;
+use std::sync::Arc;
+
+use super::core::CompfsCore;
+use crate::raw::*;
+use crate::*;
+
+#[derive(Debug)]
+pub struct CompfsLister {
+    core: Arc<CompfsCore>,
+    root: Option<String>,
+    read_dir: Option<ReadDir>,
+}
+
+impl CompfsLister {
+    pub(super) fn new(core: Arc<CompfsCore>, root: &Path, read_dir: ReadDir) -> Self {
+        let root = normalize(root, &core.root);
+        Self {
+            core,
+            root: Some(root),
+            read_dir: Some(read_dir),
+        }
+    }
+}
+
+fn normalize(path: &Path, root: &Path) -> String {
+    normalize_path(
+        &path
+            .strip_prefix(root)
+            .expect("cannot fail because the prefix is iterated")
+            .to_string_lossy()
+            .replace('\\', "/"),
+    )
+}
+
+fn next_entry(read_dir: &mut ReadDir, root: &Path) -> std::io::Result<Option<oio::Entry>> {
+    let Some(entry) = read_dir.next().transpose()? else {
+        return Ok(None);
+    };
+    let path = entry.path();
+    let rel_path = normalize(&path, root);
+
+    let file_type = entry.file_type()?;
+
+    let entry = if file_type.is_file() {
+        oio::Entry::new(&rel_path, Metadata::new(EntryMode::FILE))
+    } else if file_type.is_dir() {
+        oio::Entry::new(&format!("{rel_path}/"), Metadata::new(EntryMode::DIR))
+    } else {
+        oio::Entry::new(&rel_path, Metadata::new(EntryMode::Unknown))
+    };
+
+    Ok(Some(entry))
+}
+
+impl oio::List for CompfsLister {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        if let Some(root) = self.root.take() {
+            return Ok(Some(oio::Entry::new(
+                &format!("{root}/"),
+                Metadata::new(EntryMode::DIR),
+            )));
+        }
+        let Some(mut read_dir) = self.read_dir.take() else {
+            return Ok(None);
+        };
+        let root = self.core.root.clone();
+        let (entry, read_dir) = self
+            .core
+            .exec_blocking(move || {
+                let entry = next_entry(&mut read_dir, &root).map_err(new_std_io_error);
+                (entry, read_dir)
+            })
+            .await?;
+        if !matches!(entry, Ok(None)) {
+            self.read_dir = Some(read_dir);
+        }
+        entry
+    }
+}