opendal 0.1.6.pre.rc.1

data/core/src/services/vercel_artifacts/core.rs

@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use http::header;
+use http::Request;
+use http::Response;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct VercelArtifactsCore {
+    pub info: Arc<AccessorInfo>,
+    pub(crate) access_token: String,
+}
+
+impl Debug for VercelArtifactsCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("VercelArtifactsCore");
+        de.field("access_token", &self.access_token);
+        de.finish()
+    }
+}
+
+impl VercelArtifactsCore {
+    pub(crate) async fn vercel_artifacts_get(
+        &self,
+        hash: &str,
+        range: BytesRange,
+        _: &OpRead,
+    ) -> Result<Response<HttpBody>> {
+        let url: String = format!(
+            "https://api.vercel.com/v8/artifacts/{}",
+            percent_encode_path(hash)
+        );
+
+        let mut req = Request::get(&url);
+
+        if !range.is_full() {
+            req = req.header(header::RANGE, range.to_header());
+        }
+
+        let auth_header_content = format!("Bearer {}", self.access_token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        req = req.extension(Operation::Read);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub(crate) async fn vercel_artifacts_put(
+        &self,
+        hash: &str,
+        size: u64,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let url = format!(
+            "https://api.vercel.com/v8/artifacts/{}",
+            percent_encode_path(hash)
+        );
+
+        let mut req = Request::put(&url);
+
+        let auth_header_content = format!("Bearer {}", self.access_token);
+        req = req.header(header::CONTENT_TYPE, "application/octet-stream");
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        req = req.header(header::CONTENT_LENGTH, size);
+
+        req = req.extension(Operation::Write);
+
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub(crate) async fn vercel_artifacts_stat(&self, hash: &str) -> Result<Response<Buffer>> {
+        let url = format!(
+            "https://api.vercel.com/v8/artifacts/{}",
+            percent_encode_path(hash)
+        );
+
+        let mut req = Request::head(&url);
+
+        let auth_header_content = format!("Bearer {}", self.access_token);
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        req = req.header(header::CONTENT_LENGTH, 0);
+
+        req = req.extension(Operation::Stat);
+
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+}

data/core/src/services/vercel_artifacts/docs.md

@@ -0,0 +1,40 @@
+## Capabilities
+
+This service can be used to:
+
+- [ ] stat
+- [x] read
+- [x] write
+- [x] create_dir
+- [x] delete
+- [ ] ~~copy~~
+- [ ] ~~rename~~
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configuration
+
+- `access_token`: set the access_token for Rest API
+
+You can refer to [`VercelArtifactsBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```no_run
+use anyhow::Result;
+use opendal::services::VercelArtifacts;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create backend builder
+    let mut builder = VercelArtifacts::default()
+        .access_token("xxx");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/vercel_artifacts/error.rs

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::Response;
+use http::StatusCode;
+
+use crate::raw::*;
+use crate::*;
+
+/// Parse error response into Error.
+pub(super) fn parse_error(response: Response<Buffer>) -> Error {
+    let (parts, body) = response.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/src/services/vercel_artifacts/mod.rs

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for vercel_artifacts service.
+#[cfg(feature = "services-vercel-artifacts")]
+pub(super) const DEFAULT_SCHEME: &str = "vercel_artifacts";
+#[cfg(feature = "services-vercel-artifacts")]
+mod backend;
+#[cfg(feature = "services-vercel-artifacts")]
+mod core;
+#[cfg(feature = "services-vercel-artifacts")]
+mod error;
+#[cfg(feature = "services-vercel-artifacts")]
+mod writer;
+
+#[cfg(feature = "services-vercel-artifacts")]
+mod builder;
+#[cfg(feature = "services-vercel-artifacts")]
+pub use builder::VercelArtifactsBuilder as VercelArtifacts;
+
+mod config;
+pub use config::VercelArtifactsConfig;

data/core/src/services/vercel_artifacts/writer.rs

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::VercelArtifactsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct VercelArtifactsWriter {
+    core: Arc<VercelArtifactsCore>,
+    _op: OpWrite,
+
+    path: String,
+}
+
+impl VercelArtifactsWriter {
+    pub fn new(core: Arc<VercelArtifactsCore>, op: OpWrite, path: String) -> Self {
+        VercelArtifactsWriter {
+            core,
+            _op: op,
+            path,
+        }
+    }
+}
+
+impl oio::OneShotWrite for VercelArtifactsWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let response = self
+            .core
+            .vercel_artifacts_put(self.path.as_str(), bs.len() as u64, bs)
+            .await?;
+
+        let status = response.status();
+
+        match status {
+            StatusCode::OK | StatusCode::ACCEPTED => Ok(Metadata::default()),
+            _ => Err(parse_error(response)),
+        }
+    }
+}

data/core/src/services/vercel_blob/backend.rs

@@ -0,0 +1,251 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use log::debug;
+
+use super::core::parse_blob;
+use super::core::Blob;
+use super::core::VercelBlobCore;
+use super::delete::VercelBlobDeleter;
+use super::error::parse_error;
+use super::lister::VercelBlobLister;
+use super::writer::VercelBlobWriter;
+use super::writer::VercelBlobWriters;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::VercelBlobConfig;
+use crate::*;
+impl Configurator for VercelBlobConfig {
+    type Builder = VercelBlobBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        VercelBlobBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// [VercelBlob](https://vercel.com/docs/storage/vercel-blob) services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct VercelBlobBuilder {
+    config: VercelBlobConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for VercelBlobBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("VercelBlobBuilder");
+
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl VercelBlobBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Vercel Blob token.
+    ///
+    /// Get from Vercel environment variable `BLOB_READ_WRITE_TOKEN`.
+    /// It is required.
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+}
+
+impl Builder for VercelBlobBuilder {
+    type Config = VercelBlobConfig;
+
+    /// Builds the backend and returns the result of VercelBlobBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.clone().unwrap_or_default());
+        debug!("backend use root {}", &root);
+
+        // Handle token.
+        let Some(token) = self.config.token.clone() else {
+            return Err(Error::new(ErrorKind::ConfigInvalid, "token is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::VercelBlob));
+        };
+
+        Ok(VercelBlobBackend {
+            core: Arc::new(VercelBlobCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_root(&root)
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            read: true,
+
+                            write: true,
+                            write_can_empty: true,
+                            write_can_multi: true,
+                            write_multi_min_size: Some(5 * 1024 * 1024),
+
+                            copy: true,
+
+                            list: true,
+                            list_with_limit: true,
+
+                            delete: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+
+                    // allow deprecated api here for compatibility
+                    #[allow(deprecated)]
+                    if let Some(client) = self.http_client {
+                        am.update_http_client(|_| client);
+                    }
+
+                    am.into()
+                },
+                root,
+                token,
+            }),
+        })
+    }
+}
+
+/// Backend for VercelBlob services.
+#[derive(Debug, Clone)]
+pub struct VercelBlobBackend {
+    core: Arc<VercelBlobCore>,
+}
+
+impl Access for VercelBlobBackend {
+    type Reader = HttpBody;
+    type Writer = VercelBlobWriters;
+    type Lister = oio::PageLister<VercelBlobLister>;
+    type Deleter = oio::OneShotDeleter<VercelBlobDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let resp = self.core.head(path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let bs = resp.into_body();
+
+                let resp: Blob =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                parse_blob(&resp).map(RpStat::new)
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.download(path, args.range(), &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        let concurrent = args.concurrent();
+        let writer = VercelBlobWriter::new(self.core.clone(), args, path.to_string());
+
+        let w = oio::MultipartWriter::new(self.core.info.clone(), writer, concurrent);
+
+        Ok((RpWrite::default(), w))
+    }
+
+    async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result<RpCopy> {
+        let resp = self.core.copy(from, to).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(RpCopy::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = VercelBlobLister::new(self.core.clone(), path, args.limit());
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(VercelBlobDeleter::new(self.core.clone())),
+        ))
+    }
+}

data/core/src/services/vercel_blob/config.rs

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for VercelBlob services support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct VercelBlobConfig {
+    /// root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub root: Option<String>,
+    /// vercel blob token.
+    pub token: Option<String>,
+}
+
+impl Debug for VercelBlobConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Config");
+
+        ds.field("root", &self.root);
+
+        ds.finish()
+    }
+}