opendal 0.1.6.pre.rc.1

data/core/src/services/cos/writer.rs

@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::HeaderMap;
+use http::HeaderValue;
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub type CosWriters = TwoWays<oio::MultipartWriter<CosWriter>, oio::AppendWriter<CosWriter>>;
+
+pub struct CosWriter {
+    core: Arc<CosCore>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl CosWriter {
+    pub fn new(core: Arc<CosCore>, path: &str, op: OpWrite) -> Self {
+        CosWriter {
+            core,
+            path: path.to_string(),
+            op,
+        }
+    }
+
+    fn parse_metadata(headers: &HeaderMap<HeaderValue>) -> Result<Metadata> {
+        let mut meta = Metadata::default();
+        if let Some(etag) = parse_etag(headers)? {
+            meta.set_etag(etag);
+        }
+        if let Some(md5) = parse_content_md5(headers)? {
+            meta.set_content_md5(md5);
+        }
+        if let Some(version) = parse_header_to_str(headers, constants::X_COS_VERSION_ID)? {
+            if version != "null" {
+                meta.set_version(version);
+            }
+        }
+
+        Ok(meta)
+    }
+}
+
+impl oio::MultipartWrite for CosWriter {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .cos_put_object_request(&self.path, Some(size), &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let meta = Self::parse_metadata(resp.headers())?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self
+            .core
+            .cos_initiate_multipart_upload(&self.path, &self.op)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let result: InitiateMultipartUploadResult =
+                    quick_xml::de::from_reader(bytes::Buf::reader(bs))
+                        .map_err(new_xml_deserialize_error)?;
+
+                Ok(result.upload_id)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<oio::MultipartPart> {
+        // COS requires part number must between [1..=10000]
+        let part_number = part_number + 1;
+
+        let resp = self
+            .core
+            .cos_upload_part_request(&self.path, upload_id, part_number, size, body)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let etag = parse_etag(resp.headers())?
+                    .ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::Unexpected,
+                            "ETag not present in returning response",
+                        )
+                    })?
+                    .to_string();
+
+                Ok(oio::MultipartPart {
+                    part_number,
+                    etag,
+                    checksum: None,
+                })
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        parts: &[oio::MultipartPart],
+    ) -> Result<Metadata> {
+        let parts = parts
+            .iter()
+            .map(|p| CompleteMultipartUploadRequestPart {
+                part_number: p.part_number,
+                etag: p.etag.clone(),
+            })
+            .collect();
+
+        let mut resp = self
+            .core
+            .cos_complete_multipart_upload(&self.path, upload_id, parts)
+            .await?;
+
+        let mut meta = Self::parse_metadata(resp.headers())?;
+
+        let result: CompleteMultipartUploadResult =
+            quick_xml::de::from_reader(resp.body_mut().reader())
+                .map_err(new_xml_deserialize_error)?;
+        meta.set_etag(&result.etag);
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_part(&self, upload_id: &str) -> Result<()> {
+        let resp = self
+            .core
+            .cos_abort_multipart_upload(&self.path, upload_id)
+            .await?;
+        match resp.status() {
+            // cos returns code 204 if abort succeeds.
+            // Reference: https://www.tencentcloud.com/document/product/436/7740
+            StatusCode::NO_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}
+
+impl oio::AppendWrite for CosWriter {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self
+            .core
+            .cos_head_object(&self.path, &OpStat::default())
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let content_length = parse_content_length(resp.headers())?.ok_or_else(|| {
+                    Error::new(
+                        ErrorKind::Unexpected,
+                        "Content-Length not present in returning response",
+                    )
+                })?;
+                Ok(content_length)
+            }
+            StatusCode::NOT_FOUND => Ok(0),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        let mut req = self
+            .core
+            .cos_append_object_request(&self.path, offset, size, &self.op, body)?;
+
+        self.core.sign(&mut req).await?;
+
+        let resp = self.core.send(req).await?;
+
+        let meta = Self::parse_metadata(resp.headers())?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(meta),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/d1/backend.rs

@@ -0,0 +1,330 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use http::header;
+use http::Request;
+use http::StatusCode;
+use serde_json::Value;
+
+use super::error::parse_error;
+use super::model::D1Response;
+use crate::raw::adapters::kv;
+use crate::raw::*;
+use crate::services::D1Config;
+use crate::ErrorKind;
+use crate::*;
+
+impl Configurator for D1Config {
+    type Builder = D1Builder;
+    fn into_builder(self) -> Self::Builder {
+        D1Builder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct D1Builder {
+    config: D1Config,
+
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for D1Builder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("D1Builder")
+            .field("config", &self.config)
+            .finish()
+    }
+}
+
+impl D1Builder {
+    /// Set api token for the cloudflare d1 service.
+    ///
+    /// create a api token from [here](https://dash.cloudflare.com/profile/api-tokens)
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+
+    /// Set the account identifier for the cloudflare d1 service.
+    ///
+    /// get the account identifier from Workers & Pages -> Overview -> Account ID
+    /// If not specified, it will return an error when building.
+    pub fn account_id(mut self, account_id: &str) -> Self {
+        if !account_id.is_empty() {
+            self.config.account_id = Some(account_id.to_string());
+        }
+        self
+    }
+
+    /// Set the database identifier for the cloudflare d1 service.
+    ///
+    /// get the database identifier from Workers & Pages -> D1 -> [Your Database] -> Database ID
+    /// If not specified, it will return an error when building.
+    pub fn database_id(mut self, database_id: &str) -> Self {
+        if !database_id.is_empty() {
+            self.config.database_id = Some(database_id.to_string());
+        }
+        self
+    }
+
+    /// set the working directory, all operations will be performed under it.
+    ///
+    /// default: "/"
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the table name of the d1 service to read/write.
+    ///
+    /// If not specified, it will return an error when building.
+    pub fn table(mut self, table: &str) -> Self {
+        if !table.is_empty() {
+            self.config.table = Some(table.to_owned());
+        }
+        self
+    }
+
+    /// Set the key field name of the d1 service to read/write.
+    ///
+    /// Default to `key` if not specified.
+    pub fn key_field(mut self, key_field: &str) -> Self {
+        if !key_field.is_empty() {
+            self.config.key_field = Some(key_field.to_string());
+        }
+        self
+    }
+
+    /// Set the value field name of the d1 service to read/write.
+    ///
+    /// Default to `value` if not specified.
+    pub fn value_field(mut self, value_field: &str) -> Self {
+        if !value_field.is_empty() {
+            self.config.value_field = Some(value_field.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for D1Builder {
+    type Config = D1Config;
+
+    fn build(self) -> Result<impl Access> {
+        let mut authorization = None;
+        let config = self.config;
+
+        if let Some(token) = config.token {
+            authorization = Some(format_authorization_by_bearer(&token)?)
+        }
+
+        let Some(account_id) = config.account_id else {
+            return Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "account_id is required",
+            ));
+        };
+
+        let Some(database_id) = config.database_id.clone() else {
+            return Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "database_id is required",
+            ));
+        };
+
+        let client = if let Some(client) = self.http_client {
+            client
+        } else {
+            HttpClient::new().map_err(|err| {
+                err.with_operation("Builder::build")
+                    .with_context("service", Scheme::D1)
+            })?
+        };
+
+        let Some(table) = config.table.clone() else {
+            return Err(Error::new(ErrorKind::ConfigInvalid, "table is required"));
+        };
+
+        let key_field = config
+            .key_field
+            .clone()
+            .unwrap_or_else(|| "key".to_string());
+
+        let value_field = config
+            .value_field
+            .clone()
+            .unwrap_or_else(|| "value".to_string());
+
+        let root = normalize_root(
+            config
+                .root
+                .clone()
+                .unwrap_or_else(|| "/".to_string())
+                .as_str(),
+        );
+        Ok(D1Backend::new(Adapter {
+            authorization,
+            account_id,
+            database_id,
+            client,
+            table,
+            key_field,
+            value_field,
+        })
+        .with_normalized_root(root))
+    }
+}
+
+pub type D1Backend = kv::Backend<Adapter>;
+
+#[derive(Clone)]
+pub struct Adapter {
+    authorization: Option<String>,
+    account_id: String,
+    database_id: String,
+
+    client: HttpClient,
+    table: String,
+    key_field: String,
+    value_field: String,
+}
+
+impl Debug for Adapter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("D1Adapter");
+        ds.field("table", &self.table);
+        ds.field("key_field", &self.key_field);
+        ds.field("value_field", &self.value_field);
+        ds.finish()
+    }
+}
+
+impl Adapter {
+    fn create_d1_query_request(&self, sql: &str, params: Vec<Value>) -> Result<Request<Buffer>> {
+        let p = format!(
+            "/accounts/{}/d1/database/{}/query",
+            self.account_id, self.database_id
+        );
+        let url: String = format!(
+            "{}{}",
+            "https://api.cloudflare.com/client/v4",
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::post(&url);
+        if let Some(auth) = &self.authorization {
+            req = req.header(header::AUTHORIZATION, auth);
+        }
+        req = req.header(header::CONTENT_TYPE, "application/json");
+
+        let json = serde_json::json!({
+            "sql": sql,
+            "params": params,
+        });
+
+        let body = serde_json::to_vec(&json).map_err(new_json_serialize_error)?;
+        req.body(Buffer::from(body))
+            .map_err(new_request_build_error)
+    }
+}
+
+impl kv::Adapter for Adapter {
+    type Scanner = ();
+
+    fn info(&self) -> kv::Info {
+        kv::Info::new(
+            Scheme::D1,
+            &self.table,
+            Capability {
+                read: true,
+                write: true,
+                // Cloudflare D1 supports 1MB as max in write_total.
+                // refer to https://developers.cloudflare.com/d1/platform/limits/
+                write_total_max_size: Some(1000 * 1000),
+                shared: true,
+                ..Default::default()
+            },
+        )
+    }
+
+    async fn get(&self, path: &str) -> Result<Option<Buffer>> {
+        let query = format!(
+            "SELECT {} FROM {} WHERE {} = ? LIMIT 1",
+            self.value_field, self.table, self.key_field
+        );
+        let req = self.create_d1_query_request(&query, vec![path.into()])?;
+
+        let resp = self.client.send(req).await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                let body = resp.into_body();
+                let bs = body.to_bytes();
+                let d1_response = D1Response::parse(&bs)?;
+                Ok(d1_response.get_result(&self.value_field))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn set(&self, path: &str, value: Buffer) -> Result<()> {
+        let table = &self.table;
+        let key_field = &self.key_field;
+        let value_field = &self.value_field;
+        let query = format!(
+            "INSERT INTO {table} ({key_field}, {value_field}) \
+                VALUES (?, ?) \
+                ON CONFLICT ({key_field}) \
+                    DO UPDATE SET {value_field} = EXCLUDED.{value_field}",
+        );
+
+        let params = vec![path.into(), value.to_vec().into()];
+        let req = self.create_d1_query_request(&query, params)?;
+
+        let resp = self.client.send(req).await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn delete(&self, path: &str) -> Result<()> {
+        let query = format!("DELETE FROM {} WHERE {} = ?", self.table, self.key_field);
+        let req = self.create_d1_query_request(&query, vec![path.into()])?;
+
+        let resp = self.client.send(req).await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/d1/config.rs

@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for [Cloudflare D1](https://developers.cloudflare.com/d1) backend support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct D1Config {
+    /// Set the token of cloudflare api.
+    pub token: Option<String>,
+    /// Set the account id of cloudflare api.
+    pub account_id: Option<String>,
+    /// Set the database id of cloudflare api.
+    pub database_id: Option<String>,
+
+    /// Set the working directory of OpenDAL.
+    pub root: Option<String>,
+    /// Set the table of D1 Database.
+    pub table: Option<String>,
+    /// Set the key field of D1 Database.
+    pub key_field: Option<String>,
+    /// Set the value field of D1 Database.
+    pub value_field: Option<String>,
+}
+
+impl Debug for D1Config {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("D1Config");
+        ds.field("root", &self.root);
+        ds.field("table", &self.table);
+        ds.field("key_field", &self.key_field);
+        ds.field("value_field", &self.value_field);
+        ds.finish_non_exhaustive()
+    }
+}

data/core/src/services/d1/docs.md

@@ -0,0 +1,48 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] copy
+- [ ] rename
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `root`: Set the working directory of `OpenDAL`
+- `token`: Set the token of cloudflare api
+- `account_id`: Set the account id of cloudflare api
+- `database_id`: Set the database id of cloudflare api
+- `table`: Set the table of D1 Database
+- `key_field`: Set the key field of D1 Database
+- `value_field`: Set the value field of D1 Database
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::D1;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let mut builder = D1::default()
+        .token("token")
+        .account_id("account_id")
+        .database_id("database_id")
+        .table("table")
+        .key_field("key_field")
+        .value_field("value_field");
+
+    let op = Operator::new(builder)?.finish();
+    Ok(())
+}
+```