opendal 0.1.6.pre.rc.1

data/core/src/services/pcloud/lister.rs

@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+pub struct PcloudLister {
+    core: Arc<PcloudCore>,
+
+    path: String,
+}
+
+impl PcloudLister {
+    pub(super) fn new(core: Arc<PcloudCore>, path: &str) -> Self {
+        PcloudLister {
+            core,
+            path: path.to_string(),
+        }
+    }
+}
+
+impl oio::PageList for PcloudLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self.core.list_folder(&self.path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let resp: ListFolderResponse = serde_json::from_reader(bs.clone().reader())
+                    .map_err(new_json_deserialize_error)?;
+                let result = resp.result;
+
+                if result == 2005 {
+                    ctx.done = true;
+                    return Ok(());
+                }
+
+                if result != 0 {
+                    return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")));
+                }
+
+                if let Some(metadata) = resp.metadata {
+                    if let Some(contents) = metadata.contents {
+                        for content in contents {
+                            let path = if content.isfolder {
+                                format!("{}/", content.path.clone())
+                            } else {
+                                content.path.clone()
+                            };
+
+                            let md = parse_list_metadata(content)?;
+                            let path = build_rel_path(&self.core.root, &path);
+
+                            ctx.entries.push_back(Entry::new(&path, md))
+                        }
+                    }
+
+                    ctx.done = true;
+                    return Ok(());
+                }
+
+                Err(Error::new(
+                    ErrorKind::Unexpected,
+                    String::from_utf8_lossy(&bs.to_bytes()),
+                ))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/pcloud/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for pcloud service.
+#[cfg(feature = "services-pcloud")]
+pub(super) const DEFAULT_SCHEME: &str = "pcloud";
+#[cfg(feature = "services-pcloud")]
+mod core;
+#[cfg(feature = "services-pcloud")]
+mod delete;
+#[cfg(feature = "services-pcloud")]
+mod error;
+#[cfg(feature = "services-pcloud")]
+mod lister;
+#[cfg(feature = "services-pcloud")]
+mod writer;
+
+#[cfg(feature = "services-pcloud")]
+mod backend;
+#[cfg(feature = "services-pcloud")]
+pub use backend::PcloudBuilder as Pcloud;
+
+mod config;
+pub use config::PcloudConfig;

data/core/src/services/pcloud/writer.rs

@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::PcloudCore;
+use super::error::parse_error;
+use super::error::PcloudError;
+use crate::raw::*;
+use crate::*;
+
+pub type PcloudWriters = oio::OneShotWriter<PcloudWriter>;
+
+pub struct PcloudWriter {
+    core: Arc<PcloudCore>,
+    path: String,
+}
+
+impl PcloudWriter {
+    pub fn new(core: Arc<PcloudCore>, path: String) -> Self {
+        PcloudWriter { core, path }
+    }
+}
+
+impl oio::OneShotWrite for PcloudWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        self.core.ensure_dir_exists(&self.path).await?;
+
+        let resp = self.core.upload_file(&self.path, bs).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+                let resp: PcloudError =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+                let result = resp.result;
+
+                if result != 0 {
+                    return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}")));
+                }
+
+                Ok(Metadata::default())
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/persy/backend.rs

@@ -0,0 +1,226 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::str;
+
+use persy;
+
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::PersyConfig;
+use crate::Builder;
+use crate::Error;
+use crate::ErrorKind;
+use crate::Scheme;
+use crate::*;
+
+impl Configurator for PersyConfig {
+    type Builder = PersyBuilder;
+    fn into_builder(self) -> Self::Builder {
+        PersyBuilder { config: self }
+    }
+}
+
+/// persy service support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Debug)]
+pub struct PersyBuilder {
+    config: PersyConfig,
+}
+
+impl PersyBuilder {
+    /// Set the path to the persy data directory. Will create if not exists.
+    pub fn datafile(mut self, path: &str) -> Self {
+        self.config.datafile = Some(path.into());
+        self
+    }
+
+    /// Set the name of the persy segment. Will create if not exists.
+    pub fn segment(mut self, path: &str) -> Self {
+        self.config.segment = Some(path.into());
+        self
+    }
+
+    /// Set the name of the persy index. Will create if not exists.
+    pub fn index(mut self, path: &str) -> Self {
+        self.config.index = Some(path.into());
+        self
+    }
+}
+
+impl Builder for PersyBuilder {
+    type Config = PersyConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let datafile_path = self.config.datafile.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "datafile is required but not set")
+                .with_context("service", Scheme::Persy)
+        })?;
+
+        let segment_name = self.config.segment.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "segment is required but not set")
+                .with_context("service", Scheme::Persy)
+        })?;
+
+        let segment = segment_name.clone();
+
+        let index_name = self.config.index.ok_or_else(|| {
+            Error::new(ErrorKind::ConfigInvalid, "index is required but not set")
+                .with_context("service", Scheme::Persy)
+        })?;
+
+        let index = index_name.clone();
+
+        let persy = persy::OpenOptions::new()
+            .create(true)
+            .prepare_with(move |p| init(p, &segment_name, &index_name))
+            .open(&datafile_path)
+            .map_err(|e| {
+                Error::new(ErrorKind::ConfigInvalid, "open db")
+                    .with_context("service", Scheme::Persy)
+                    .with_context("datafile", datafile_path.clone())
+                    .set_source(e)
+            })?;
+
+        // This function will only be called on database creation
+        fn init(
+            persy: &persy::Persy,
+            segment_name: &str,
+            index_name: &str,
+        ) -> Result<(), Box<dyn std::error::Error>> {
+            let mut tx = persy.begin()?;
+
+            if !tx.exists_segment(segment_name)? {
+                tx.create_segment(segment_name)?;
+            }
+            if !tx.exists_index(index_name)? {
+                tx.create_index::<String, persy::PersyId>(index_name, persy::ValueMode::Replace)?;
+            }
+
+            let prepared = tx.prepare()?;
+            prepared.commit()?;
+
+            Ok(())
+        }
+
+        Ok(PersyBackend::new(Adapter {
+            datafile: datafile_path,
+            segment,
+            index,
+            persy,
+        }))
+    }
+}
+
+/// Backend for persy services.
+pub type PersyBackend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    datafile: String,
+    segment: String,
+    index: String,
+    persy: persy::Persy,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Adapter");
+        ds.field("path", &self.datafile);
+        ds.field("segment", &self.segment);
+        ds.field("index", &self.index);
+        ds.finish()
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::Persy,
+            &self.datafile,
+            Capability {
+                read: true,
+                write: true,
+                delete: true,
+                shared: false,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let mut read_id = self
+            .persy
+            .get::<String, persy::PersyId>(&self.index, &path.to_string())
+            .map_err(parse_error)?;
+        if let Some(id) = read_id.next() {
+            let value = self.persy.read(&self.segment, &id).map_err(parse_error)?;
+            return Ok(value.map(Buffer::from));
+        }
+
+        Ok(None)
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let mut tx = self.persy.begin().map_err(parse_error)?;
+        let id = tx
+            .insert(&self.segment, &value.to_vec())
+            .map_err(parse_error)?;
+
+        tx.put::<String, persy::PersyId>(&self.index, path.to_string(), id)
+            .map_err(parse_error)?;
+        let prepared = tx.prepare().map_err(parse_error)?;
+        prepared.commit().map_err(parse_error)?;
+
+        Ok(())
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let mut delete_id = self
+            .persy
+            .get::<String, persy::PersyId>(&self.index, &path.to_string())
+            .map_err(parse_error)?;
+        if let Some(id) = delete_id.next() {
+            // Begin a transaction.
+            let mut tx = self.persy.begin().map_err(parse_error)?;
+            // Delete the record.
+            tx.delete(&self.segment, &id).map_err(parse_error)?;
+            // Remove the index.
+            tx.remove::<String, persy::PersyId>(&self.index, path.to_string(), Some(id))
+                .map_err(parse_error)?;
+            // Commit the tx.
+            let prepared = tx.prepare().map_err(parse_error)?;
+            prepared.commit().map_err(parse_error)?;
+        }
+
+        Ok(())
+    }
+}
+
+fn parse_error<T: Into<persy::PersyError>>(err: persy::PE<T>) -> Error {
+    let err: persy::PersyError = err.persy_error();
+    let kind = match err {
+        persy::PersyError::RecordNotFound(_) => ErrorKind::NotFound,
+        _ => ErrorKind::Unexpected,
+    };
+
+    Error::new(kind, "error from persy").set_source(err)
+}

data/core/src/services/persy/config.rs

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for persy service support.
+#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct PersyConfig {
+    /// That path to the persy data file. The directory in the path must already exist.
+    pub datafile: Option<String>,
+    /// That name of the persy segment.
+    pub segment: Option<String>,
+    /// That name of the persy index.
+    pub index: Option<String>,
+}

data/core/src/services/persy/docs.md

@@ -0,0 +1,43 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] ~~presign~~
+- [x] blocking
+
+## Configuration
+
+- `datafile`: Set the path to the persy data file. The directory in the path must already exist.
+- `segment`: Set the name of the persy segment.
+- `index`: Set the name of the persy index.
+
+You can refer to [`PersyBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Persy;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = Persy::default()
+        .datafile("./test.persy")
+        .segment("data")
+        .index("index");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/persy/mod.rs

@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-persy")]
+mod backend;
+#[cfg(feature = "services-persy")]
+pub use backend::PersyBuilder as Persy;
+
+mod config;
+pub use config::PersyConfig;