opendal 0.1.6.pre.rc.1

data/core/src/services/cloudflare_kv/core.rs

@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use http::header;
+use http::request;
+use http::Request;
+use http::Response;
+use serde_json::json;
+
+use crate::raw::new_json_serialize_error;
+use crate::raw::percent_encode_path;
+use crate::raw::Operation;
+use crate::raw::QueryPairsWriter;
+use crate::raw::{new_request_build_error, AccessorInfo, FormDataPart, Multipart};
+use crate::services::cloudflare_kv::model::CfKvMetadata;
+use crate::{Buffer, Result};
+
+#[derive(Debug, Clone)]
+pub struct CloudflareKvCore {
+    pub api_token: String,
+    pub account_id: String,
+    pub namespace_id: String,
+    pub expiration_ttl: Option<Duration>,
+    pub info: Arc<AccessorInfo>,
+}
+
+impl CloudflareKvCore {
+    #[inline]
+    async fn send(&self, req: Request<Buffer>) -> Result<Response<Buffer>> {
+        self.info.http_client().send(req).await
+    }
+
+    fn sign(&self, req: request::Builder) -> request::Builder {
+        req.header(header::AUTHORIZATION, &self.api_token)
+    }
+
+    fn url_prefix(&self) -> String {
+        let url = format!(
+            "https://api.cloudflare.com/client/v4/accounts/{}/storage/kv/namespaces/{}",
+            self.account_id, self.namespace_id
+        );
+        url
+    }
+}
+
+impl CloudflareKvCore {
+    pub async fn metadata(&self, path: &str) -> Result<Response<Buffer>> {
+        let url = format!(
+            "{}/metadata/{}",
+            self.url_prefix(),
+            percent_encode_path(path)
+        );
+
+        let req = Request::get(url);
+        let req = self.sign(req);
+
+        let req = req
+            .extension(Operation::Stat)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn get(&self, path: &str) -> Result<Response<Buffer>> {
+        let url = format!("{}/values/{}", self.url_prefix(), percent_encode_path(path));
+        let req = Request::get(url);
+
+        let req = self.sign(req);
+
+        let req = req
+            .extension(Operation::Read)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn set(
+        &self,
+        path: &str,
+        value: Buffer,
+        metadata: CfKvMetadata,
+    ) -> Result<Response<Buffer>> {
+        let url = format!("{}/values/{}", self.url_prefix(), percent_encode_path(path));
+
+        let req = Request::put(url);
+        let req = self.sign(req);
+        let req = req.extension(Operation::Write);
+
+        let mut multipart = Multipart::new()
+            .part(FormDataPart::new("value").content(value))
+            .part(
+                FormDataPart::new("metadata")
+                    .content(serde_json::to_string(&metadata).map_err(new_json_serialize_error)?),
+            );
+
+        if let Some(expiration_ttl) = self.expiration_ttl {
+            multipart = multipart.part(
+                FormDataPart::new("expiration_ttl").content(expiration_ttl.as_secs().to_string()),
+            );
+        }
+
+        let req = multipart.apply(req)?;
+
+        self.send(req).await
+    }
+
+    pub async fn delete(&self, paths: &[String]) -> Result<Response<Buffer>> {
+        let url = format!("{}/bulk/delete", self.url_prefix());
+
+        let req = Request::post(&url);
+
+        let req = self.sign(req);
+        let req_body = &json!(paths);
+        let req = req
+            .extension(Operation::Delete)
+            .header(header::CONTENT_TYPE, "application/json")
+            .body(Buffer::from(req_body.to_string()))
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+
+    pub async fn list(
+        &self,
+        prefix: &str,
+        limit: Option<usize>,
+        cursor: Option<String>,
+    ) -> Result<Response<Buffer>> {
+        let url = format!("{}/keys", self.url_prefix());
+        let mut url = QueryPairsWriter::new(&url);
+        if let Some(cursor) = cursor {
+            if !cursor.is_empty() {
+                url = url.push("cursor", &cursor);
+            }
+        }
+        url = url.push("limit", &limit.unwrap_or(1000).to_string());
+        url = url.push("prefix", &percent_encode_path(prefix));
+
+        let req = Request::get(url.finish());
+
+        let req = self.sign(req);
+        let req = req
+            .extension(Operation::List)
+            .body(Buffer::new())
+            .map_err(new_request_build_error)?;
+
+        self.send(req).await
+    }
+}

data/core/src/services/cloudflare_kv/delete.rs

@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::oio::BatchDeleteResult;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::CfKvDeleteResponse;
+use crate::*;
+
+pub struct CloudflareKvDeleter {
+    core: Arc<CloudflareKvCore>,
+}
+
+impl CloudflareKvDeleter {
+    pub fn new(core: Arc<CloudflareKvCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::BatchDelete for CloudflareKvDeleter {
+    async fn delete_once(&self, path: String, _: OpDelete) -> Result<()> {
+        let path = build_abs_path(&self.core.info.root(), &path);
+        let resp = self
+            .core
+            .delete(&[path.trim_end_matches('/').to_string()])
+            .await?;
+
+        let status = resp.status();
+
+        if status != StatusCode::OK {
+            return Err(parse_error(resp.clone()));
+        }
+
+        let bs = resp.clone().into_body();
+        let res: CfKvDeleteResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        if !res.success {
+            return Err(parse_error(resp.clone()));
+        }
+
+        Ok(())
+    }
+
+    async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
+        let keys = batch
+            .iter()
+            .map(|path| {
+                let path = build_abs_path(&self.core.info.root(), &path.0);
+                path.trim_end_matches('/').to_string()
+            })
+            .collect::<Vec<String>>();
+
+        let resp = self.core.delete(&keys).await?;
+
+        let status = resp.status();
+
+        if status != StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+
+        let res: CfKvDeleteResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        let result = match (res.success, res.result) {
+            (true, Some(result)) => result,
+            _ => {
+                return Err(Error::new(
+                    ErrorKind::Unexpected,
+                    "cloudflare_kv delete this key failed for reason we don't know",
+                ))
+            }
+        };
+
+        let mut batched_result = BatchDeleteResult {
+            succeeded: Vec::with_capacity(result.successful_key_count),
+            failed: Vec::with_capacity(result.unsuccessful_keys.len()),
+        };
+
+        for item in batch {
+            if result.unsuccessful_keys.contains(&item.0) {
+                batched_result.failed.push((
+                    item.0,
+                    item.1,
+                    Error::new(
+                        ErrorKind::Unexpected,
+                        "cloudflare_kv delete this key failed for reason we don't know",
+                    ),
+                ));
+            } else {
+                batched_result.succeeded.push(item);
+            }
+        }
+
+        Ok(batched_result)
+    }
+}

data/core/src/services/cloudflare_kv/docs.md

@@ -0,0 +1,21 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `api_token`: Set the token of cloudflare api
+- `account_id`: Set the account identifier of cloudflare
+- `namespace_id`: Set the namespace identifier of d1

data/core/src/services/cloudflare_kv/error.rs

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use serde_json::de;
+
+use super::model::CfKvError;
+use super::model::CfKvResponse;
+use crate::raw::*;
+use crate::*;
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (mut kind, mut retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        // Some services (like owncloud) return 403 while file locked.
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
+        // Allowing retry for resource locked.
+        StatusCode::LOCKED => (ErrorKind::Unexpected, true),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, err) = de::from_reader::<_, CfKvResponse>(bs.clone().reader())
+        .map(|err| (format!("{err:?}"), Some(err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    if let Some(err) = err {
+        (kind, retryable) = parse_cfkv_error_code(err.errors).unwrap_or((kind, retryable));
+    }
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+pub(super) fn parse_cfkv_error_code(errors: Vec<CfKvError>) -> Option<(ErrorKind, bool)> {
+    if errors.is_empty() {
+        return None;
+    }
+
+    match errors[0].code {
+        // The request is malformed: failed to decode id.
+        7400 => Some((ErrorKind::Unexpected, false)),
+        // no such column: Xxxx.
+        7500 => Some((ErrorKind::NotFound, false)),
+        // Authentication error.
+        10000 => Some((ErrorKind::PermissionDenied, false)),
+        _ => None,
+    }
+}

data/core/src/services/cloudflare_kv/lister.rs

@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::CloudflareKvCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::services::cloudflare_kv::model::{CfKvListKey, CfKvListResponse};
+use crate::*;
+
+pub struct CloudflareKvLister {
+    core: Arc<CloudflareKvCore>,
+
+    path: String,
+    limit: Option<usize>,
+    recursive: bool,
+}
+
+impl CloudflareKvLister {
+    pub fn new(
+        core: Arc<CloudflareKvCore>,
+        path: &str,
+        recursive: bool,
+        limit: Option<usize>,
+    ) -> Self {
+        Self {
+            core,
+
+            path: path.to_string(),
+            limit,
+            recursive,
+        }
+    }
+
+    fn build_entry_for_item(&self, item: &CfKvListKey, root: &str) -> Result<oio::Entry> {
+        let metadata = item.metadata.clone();
+        let mut name = item.name.clone();
+
+        if metadata.is_dir && !name.ends_with('/') {
+            name += "/";
+        }
+
+        let mut name = name.replace(root.trim_start_matches('/'), "");
+
+        // If it is the root directory, it needs to be processed as /
+        if name.is_empty() {
+            name = "/".to_string();
+        }
+
+        let entry_metadata = if name.ends_with('/') {
+            Metadata::new(EntryMode::DIR)
+                .with_etag(build_tmp_path_of(&name))
+                .with_content_length(0)
+        } else {
+            Metadata::new(EntryMode::FILE)
+                .with_etag(metadata.etag)
+                .with_content_length(metadata.content_length as u64)
+                .with_last_modified(parse_datetime_from_rfc3339(&metadata.last_modified)?)
+        };
+
+        Ok(oio::Entry::new(&name, entry_metadata))
+    }
+
+    fn handle_non_recursive_file_list(
+        &self,
+        ctx: &mut oio::PageContext,
+        result: &[CfKvListKey],
+        root: &str,
+    ) -> Result<()> {
+        if let Some(item) = result.iter().find(|item| item.name == self.path) {
+            let entry = self.build_entry_for_item(item, root)?;
+            ctx.entries.push_back(entry);
+        } else if !result.is_empty() {
+            let path_name = self.path.replace(root.trim_start_matches('/'), "");
+            let entry = oio::Entry::new(
+                &format!("{path_name}/"),
+                Metadata::new(EntryMode::DIR)
+                    .with_etag(build_tmp_path_of(&path_name))
+                    .with_content_length(0),
+            );
+            ctx.entries.push_back(entry);
+        }
+        ctx.done = true;
+        Ok(())
+    }
+}
+
+impl oio::PageList for CloudflareKvLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let new_path = self.path.trim_end_matches('/');
+        let resp = self
+            .core
+            .list(new_path, self.limit, Some(ctx.token.clone()))
+            .await?;
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        let bs = resp.into_body();
+        let res: CfKvListResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        if !res.success {
+            return Err(Error::new(
+                ErrorKind::Unexpected,
+                "oss list this key failed for reason we don't know",
+            ));
+        }
+
+        let (token, done) = res
+            .result_info
+            .and_then(|info| info.cursor)
+            .map_or((String::new(), true), |cursor| {
+                (cursor.clone(), cursor.is_empty())
+            });
+
+        ctx.token = token;
+        ctx.done = done;
+
+        if let Some(result) = res.result {
+            let root = self.core.info.root().to_string();
+
+            if !self.path.ends_with('/') && !self.recursive {
+                self.handle_non_recursive_file_list(ctx, &result, &root)?;
+                return Ok(());
+            }
+
+            for item in result {
+                let mut name = item.name.clone();
+                if item.metadata.is_dir && !name.ends_with('/') {
+                    name += "/";
+                }
+
+                // For non-recursive listing, filter out entries not in the current directory.
+                if !self.recursive {
+                    if let Some(relative_path) = name.strip_prefix(&self.path) {
+                        if relative_path.trim_end_matches('/').contains('/') {
+                            continue;
+                        }
+                    } else if self.path != name {
+                        continue;
+                    }
+                }
+
+                let entry = self.build_entry_for_item(&item, &root)?;
+                ctx.entries.push_back(entry);
+            }
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/cloudflare_kv/mod.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-cloudflare-kv")]
+pub(super) const DEFAULT_SCHEME: &str = "cloudflare_kv";
+#[cfg(feature = "services-cloudflare-kv")]
+mod error;
+
+#[cfg(feature = "services-cloudflare-kv")]
+mod backend;
+#[cfg(feature = "services-cloudflare-kv")]
+mod core;
+#[cfg(feature = "services-cloudflare-kv")]
+mod delete;
+#[cfg(feature = "services-cloudflare-kv")]
+mod lister;
+#[cfg(feature = "services-cloudflare-kv")]
+mod model;
+#[cfg(feature = "services-cloudflare-kv")]
+mod writer;
+#[cfg(feature = "services-cloudflare-kv")]
+pub use backend::CloudflareKvBuilder as CloudflareKv;
+
+mod config;
+pub use config::CloudflareKvConfig;

data/core/src/services/cloudflare_kv/model.rs

@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvResponse {
+    pub errors: Vec<CfKvError>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct CfKvMetadata {
+    pub etag: String,
+    pub last_modified: String,
+    pub content_length: usize,
+    pub is_dir: bool,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvError {
+    pub code: i32,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvDeleteResult {
+    pub successful_key_count: usize,
+    pub unsuccessful_keys: Vec<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvDeleteResponse {
+    pub success: bool,
+    pub result: Option<CfKvDeleteResult>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListKey {
+    pub name: String,
+    pub metadata: CfKvMetadata,
+    // expiration: Option<u32>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListResultInfo {
+    // pub count: Option<usize>,
+    pub cursor: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvListResponse {
+    pub success: bool,
+    pub result_info: Option<CfKvListResultInfo>,
+    pub result: Option<Vec<CfKvListKey>>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CfKvStatResponse {
+    pub success: bool,
+    pub result: Option<CfKvMetadata>,
+}