opendal 0.1.6.pre.rc.1

data/core/src/services/upyun/error.rs

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use http::Response;
+use quick_xml::de;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// UpyunError is the error returned by upyun service.
+#[derive(Default, Debug, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct UpyunError {
+    code: i64,
+    msg: String,
+    id: String,
+}
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status.as_u16() {
+        403 => (ErrorKind::PermissionDenied, false),
+        404 => (ErrorKind::NotFound, false),
+        304 | 412 => (ErrorKind::ConditionNotMatch, false),
+        // Service like Upyun could return 499 error with a message like:
+        // Client Disconnect, we should retry it.
+        499 => (ErrorKind::Unexpected, true),
+        500 | 502 | 503 | 504 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let (message, _upyun_err) = de::from_reader::<_, UpyunError>(bs.clone().reader())
+        .map(|upyun_err| (format!("{upyun_err:?}"), Some(upyun_err)))
+        .unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod test {
+    use http::StatusCode;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_parse_error() {
+        let err_res = vec![
+            (
+                r#"{"code": 40100016, "msg": "invalid date value in header", "id": "f5b30c720ddcecc70abd2f5c1c64bde8"}"#,
+                ErrorKind::Unexpected,
+                StatusCode::UNAUTHORIZED,
+            ),
+            (
+                r#"{"code": 40300010, "msg": "file type error", "id": "f5b30c720ddcecc70abd2f5c1c64bde7"}"#,
+                ErrorKind::PermissionDenied,
+                StatusCode::FORBIDDEN,
+            ),
+        ];
+
+        for res in err_res {
+            let bs = bytes::Bytes::from(res.0);
+            let body = Buffer::from(bs);
+            let resp = Response::builder().status(res.2).body(body).unwrap();
+
+            let err = parse_error(resp);
+
+            assert_eq!(err.kind(), res.1);
+        }
+    }
+}

data/core/src/services/upyun/lister.rs

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::ListObjectsResponse;
+use super::core::UpyunCore;
+use super::error::parse_error;
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::EntryMode;
+use crate::Metadata;
+use crate::Result;
+
+pub struct UpyunLister {
+    core: Arc<UpyunCore>,
+
+    path: String,
+    limit: Option<usize>,
+}
+
+impl UpyunLister {
+    pub(super) fn new(core: Arc<UpyunCore>, path: &str, limit: Option<usize>) -> Self {
+        UpyunLister {
+            core,
+            path: path.to_string(),
+            limit,
+        }
+    }
+}
+
+impl oio::PageList for UpyunLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .list_objects(&self.path, &ctx.token, self.limit)
+            .await?;
+
+        if resp.status() == http::StatusCode::NOT_FOUND {
+            ctx.done = true;
+            return Ok(());
+        }
+
+        match resp.status() {
+            http::StatusCode::OK => {}
+            http::StatusCode::NOT_FOUND => {
+                ctx.done = true;
+                return Ok(());
+            }
+            _ => {
+                return Err(parse_error(resp));
+            }
+        }
+
+        let bs = resp.into_body();
+
+        let response: ListObjectsResponse =
+            serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        // ref https://help.upyun.com/knowledge-base/rest_api/#e88eb7e58f96e79baee5bd95e69687e4bbb6e58897e8a1a8
+        // when iter is "g2gCZAAEbmV4dGQAA2VvZg", it means the list is done.
+        ctx.done = response.iter == "g2gCZAAEbmV4dGQAA2VvZg";
+
+        ctx.token = response.iter;
+
+        for file in response.files {
+            let path = build_abs_path(&normalize_root(&self.path), &file.name);
+
+            let entry = if file.type_field == "folder" {
+                let path = format!("{path}/");
+                Entry::new(&path, Metadata::new(EntryMode::DIR))
+            } else {
+                let m = Metadata::new(EntryMode::FILE)
+                    .with_content_length(file.length)
+                    .with_content_type(file.type_field)
+                    .with_last_modified(parse_datetime_from_from_timestamp(file.last_modified)?);
+                Entry::new(&path, m)
+            };
+
+            ctx.entries.push_back(entry);
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/upyun/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for upyun service.
+#[cfg(feature = "services-upyun")]
+pub(super) const DEFAULT_SCHEME: &str = "upyun";
+#[cfg(feature = "services-upyun")]
+mod core;
+#[cfg(feature = "services-upyun")]
+mod delete;
+#[cfg(feature = "services-upyun")]
+mod error;
+#[cfg(feature = "services-upyun")]
+mod lister;
+#[cfg(feature = "services-upyun")]
+mod writer;
+
+#[cfg(feature = "services-upyun")]
+mod backend;
+#[cfg(feature = "services-upyun")]
+pub use backend::UpyunBuilder as Upyun;
+
+mod config;
+pub use config::UpyunConfig;

data/core/src/services/upyun/writer.rs

@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::constants::X_UPYUN_MULTI_UUID;
+use super::core::UpyunCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub type UpyunWriters = oio::MultipartWriter<UpyunWriter>;
+
+pub struct UpyunWriter {
+    core: Arc<UpyunCore>,
+    op: OpWrite,
+    path: String,
+}
+
+impl UpyunWriter {
+    pub fn new(core: Arc<UpyunCore>, op: OpWrite, path: String) -> Self {
+        UpyunWriter { core, op, path }
+    }
+}
+
+impl oio::MultipartWrite for UpyunWriter {
+    async fn write_once(&self, size: u64, body: Buffer) -> Result<Metadata> {
+        let req = self.core.upload(&self.path, Some(size), &self.op, body)?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn initiate_part(&self) -> Result<String> {
+        let resp = self
+            .core
+            .initiate_multipart_upload(&self.path, &self.op)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT => {
+                let id =
+                    parse_header_to_str(resp.headers(), X_UPYUN_MULTI_UUID)?.ok_or(Error::new(
+                        ErrorKind::Unexpected,
+                        format!("{X_UPYUN_MULTI_UUID} header is missing"),
+                    ))?;
+
+                Ok(id.to_string())
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn write_part(
+        &self,
+        upload_id: &str,
+        part_number: usize,
+        size: u64,
+        body: Buffer,
+    ) -> Result<oio::MultipartPart> {
+        let req = self
+            .core
+            .upload_part(&self.path, upload_id, part_number, size, body)?;
+
+        let resp = self.core.send(req).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT | StatusCode::CREATED => Ok(oio::MultipartPart {
+                part_number,
+                etag: "".to_string(),
+                checksum: None,
+            }),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn complete_part(
+        &self,
+        upload_id: &str,
+        _parts: &[oio::MultipartPart],
+    ) -> Result<Metadata> {
+        let resp = self
+            .core
+            .complete_multipart_upload(&self.path, upload_id)
+            .await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn abort_part(&self, _upload_id: &str) -> Result<()> {
+        Err(Error::new(
+            ErrorKind::Unsupported,
+            "Upyun does not support abort multipart upload",
+        ))
+    }
+}

data/core/src/services/vercel_artifacts/backend.rs

@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+
+use super::core::VercelArtifactsCore;
+use super::error::parse_error;
+use super::writer::VercelArtifactsWriter;
+use crate::raw::*;
+use crate::*;
+
+#[doc = include_str!("docs.md")]
+#[derive(Clone)]
+pub struct VercelArtifactsBackend {
+    pub core: Arc<VercelArtifactsCore>,
+}
+
+impl Debug for VercelArtifactsBackend {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("VercelArtifactsBackend")
+            .field("core", &self.core)
+            .finish()
+    }
+}
+
+impl Access for VercelArtifactsBackend {
+    type Reader = HttpBody;
+    type Writer = oio::OneShotWriter<VercelArtifactsWriter>;
+    type Lister = ();
+    type Deleter = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _args: OpStat) -> Result<RpStat> {
+        let response = self.core.vercel_artifacts_stat(path).await?;
+
+        let status = response.status();
+
+        match status {
+            StatusCode::OK => {
+                let meta = parse_into_metadata(path, response.headers())?;
+                Ok(RpStat::new(meta))
+            }
+
+            _ => Err(parse_error(response)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let response = self
+            .core
+            .vercel_artifacts_get(path, args.range(), &args)
+            .await?;
+
+        let status = response.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::new(), response.into_body()))
+            }
+            _ => {
+                let (part, mut body) = response.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        Ok((
+            RpWrite::default(),
+            oio::OneShotWriter::new(VercelArtifactsWriter::new(
+                self.core.clone(),
+                args,
+                path.to_string(),
+            )),
+        ))
+    }
+}

data/core/src/services/vercel_artifacts/builder.rs

@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use super::backend::VercelArtifactsBackend;
+use super::core::VercelArtifactsCore;
+use super::DEFAULT_SCHEME;
+use crate::raw::Access;
+use crate::raw::AccessorInfo;
+use crate::raw::HttpClient;
+use crate::services::VercelArtifactsConfig;
+use crate::*;
+impl Configurator for VercelArtifactsConfig {
+    type Builder = VercelArtifactsBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        VercelArtifactsBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// [Vercel Cache](https://vercel.com/docs/concepts/monorepos/remote-caching) backend support.
+#[doc = include_str!("docs.md")]
+#[derive(Default)]
+pub struct VercelArtifactsBuilder {
+    config: VercelArtifactsConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for VercelArtifactsBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut d = f.debug_struct("VercelArtifactsBuilder");
+        d.field("config", &self.config);
+        d.finish_non_exhaustive()
+    }
+}
+
+impl VercelArtifactsBuilder {
+    /// set the bearer access token for Vercel
+    ///
+    /// default: no access token, which leads to failure
+    pub fn access_token(mut self, access_token: &str) -> Self {
+        self.config.access_token = Some(access_token.to_string());
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, http_client: HttpClient) -> Self {
+        self.http_client = Some(http_client);
+        self
+    }
+}
+
+impl Builder for VercelArtifactsBuilder {
+    type Config = VercelArtifactsConfig;
+
+    fn build(self) -> Result<impl Access> {
+        let info = AccessorInfo::default();
+        info.set_scheme(DEFAULT_SCHEME)
+            .set_native_capability(Capability {
+                stat: true,
+
+                read: true,
+
+                write: true,
+
+                shared: true,
+
+                ..Default::default()
+            });
+
+        // allow deprecated api here for compatibility
+        #[allow(deprecated)]
+        if let Some(client) = self.http_client {
+            info.update_http_client(|_| client);
+        }
+
+        match self.config.access_token.clone() {
+            Some(access_token) => Ok(VercelArtifactsBackend {
+                core: Arc::new(VercelArtifactsCore {
+                    info: Arc::new(info),
+                    access_token,
+                }),
+            }),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")),
+        }
+    }
+}

data/core/src/services/vercel_artifacts/config.rs

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Vercel Cache support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct VercelArtifactsConfig {
+    /// The access token for Vercel.
+    pub access_token: Option<String>,
+}
+
+impl Debug for VercelArtifactsConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("VercelArtifactsConfig")
+            .field("access_token", &"<redacted>")
+            .finish()
+    }
+}