opendal 0.1.6.pre.rc.1

data/core/src/services/http/config.rs

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for Http service support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct HttpConfig {
+    /// endpoint of this backend
+    pub endpoint: Option<String>,
+    /// username of this backend
+    pub username: Option<String>,
+    /// password of this backend
+    pub password: Option<String>,
+    /// token of this backend
+    pub token: Option<String>,
+    /// root of this backend
+    pub root: Option<String>,
+}
+
+impl Debug for HttpConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("HttpConfig");
+        de.field("endpoint", &self.endpoint);
+        de.field("root", &self.root);
+
+        de.finish_non_exhaustive()
+    }
+}

data/core/src/services/http/core.rs

@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::header;
+use http::header::IF_MATCH;
+use http::header::IF_NONE_MATCH;
+use http::Request;
+use http::Response;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct HttpCore {
+    pub info: Arc<AccessorInfo>,
+
+    pub endpoint: String,
+    pub root: String,
+
+    pub authorization: Option<String>,
+}
+
+impl Debug for HttpCore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HttpCore")
+            .field("endpoint", &self.endpoint)
+            .field("root", &self.root)
+            .finish()
+    }
+}
+
+impl HttpCore {
+    pub fn has_authorization(&self) -> bool {
+        self.authorization.is_some()
+    }
+
+    pub fn http_get_request(
+        &self,
+        path: &str,
+        range: BytesRange,
+        args: &OpRead,
+    ) -> Result<Request<Buffer>> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
+
+        let mut req = Request::get(&url);
+
+        if let Some(if_match) = args.if_match() {
+            req = req.header(IF_MATCH, if_match);
+        }
+
+        if let Some(if_none_match) = args.if_none_match() {
+            req = req.header(IF_NONE_MATCH, if_none_match);
+        }
+
+        if let Some(auth) = &self.authorization {
+            req = req.header(header::AUTHORIZATION, auth.clone())
+        }
+
+        if !range.is_full() {
+            req = req.header(header::RANGE, range.to_header());
+        }
+
+        let req = req.extension(Operation::Read);
+
+        req.body(Buffer::new()).map_err(new_request_build_error)
+    }
+
+    pub async fn http_get(
+        &self,
+        path: &str,
+        range: BytesRange,
+        args: &OpRead,
+    ) -> Result<Response<HttpBody>> {
+        let req = self.http_get_request(path, range, args)?;
+        self.info.http_client().fetch(req).await
+    }
+
+    pub fn http_head_request(&self, path: &str, args: &OpStat) -> Result<Request<Buffer>> {
+        let p = build_rooted_abs_path(&self.root, path);
+
+        let url = format!("{}{}", self.endpoint, percent_encode_path(&p));
+
+        let mut req = Request::head(&url);
+
+        if let Some(if_match) = args.if_match() {
+            req = req.header(IF_MATCH, if_match);
+        }
+
+        if let Some(if_none_match) = args.if_none_match() {
+            req = req.header(IF_NONE_MATCH, if_none_match);
+        }
+
+        if let Some(auth) = &self.authorization {
+            req = req.header(header::AUTHORIZATION, auth.clone())
+        }
+
+        let req = req.extension(Operation::Stat);
+
+        req.body(Buffer::new()).map_err(new_request_build_error)
+    }
+
+    pub async fn http_head(&self, path: &str, args: &OpStat) -> Result<Response<Buffer>> {
+        let req = self.http_head_request(path, args)?;
+        self.info.http_client().send(req).await
+    }
+}

data/core/src/services/http/docs.md

@@ -0,0 +1,45 @@
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [ ] ~~write~~
+- [ ] ~~create_dir~~
+- [ ] ~~delete~~
+- [ ] ~~copy~~
+- [ ] ~~rename~~
+- [ ] ~~list~~
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Notes
+
+Only `read` and `stat` are supported. We can use this service to visit any
+HTTP Server like nginx, caddy.
+
+## Configuration
+
+- `endpoint`: set the endpoint for http
+- `root`: Set the work directory for backend
+
+You can refer to [`HttpBuilder`]'s docs for more information
+
+## Example
+
+### Via Builder
+
+```rust,no_run
+use anyhow::Result;
+use opendal::services::Http;
+use opendal::Operator;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // create http backend builder
+    let mut builder = Http::default().endpoint("127.0.0.1");
+
+    let op: Operator = Operator::new(builder)?.finish();
+    Ok(())
+}
+```

data/core/src/services/http/error.rs

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use http::Response;
+use http::StatusCode;
+
+use crate::raw::*;
+use crate::*;
+
+/// Parse error response into Error.
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => {
+            (ErrorKind::ConditionNotMatch, false)
+        }
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = String::from_utf8_lossy(&bs);
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}

data/core/src/services/http/mod.rs

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for http service.
+#[cfg(feature = "services-http")]
+pub(super) const DEFAULT_SCHEME: &str = "http";
+#[cfg(feature = "services-http")]
+mod error;
+
+#[cfg(feature = "services-http")]
+mod backend;
+#[cfg(feature = "services-http")]
+mod core;
+#[cfg(feature = "services-http")]
+pub use backend::HttpBuilder as Http;
+
+mod config;
+pub use config::HttpConfig;

data/core/src/services/huggingface/backend.rs

@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use bytes::Buf;
+use http::Response;
+use http::StatusCode;
+use log::debug;
+
+use super::core::HuggingfaceCore;
+use super::core::HuggingfaceStatus;
+use super::error::parse_error;
+use super::lister::HuggingfaceLister;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::HuggingfaceConfig;
+use crate::*;
+impl Configurator for HuggingfaceConfig {
+    type Builder = HuggingfaceBuilder;
+    fn into_builder(self) -> Self::Builder {
+        HuggingfaceBuilder { config: self }
+    }
+}
+
+/// [Huggingface](https://huggingface.co/docs/huggingface_hub/package_reference/hf_api)'s API support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Clone)]
+pub struct HuggingfaceBuilder {
+    config: HuggingfaceConfig,
+}
+
+impl Debug for HuggingfaceBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Builder");
+
+        ds.field("config", &self.config);
+        ds.finish()
+    }
+}
+
+impl HuggingfaceBuilder {
+    /// Set repo type of this backend. Default is model.
+    ///
+    /// Available values:
+    /// - model
+    /// - dataset
+    ///
+    /// Currently, only models and datasets are supported.
+    /// [Reference](https://huggingface.co/docs/hub/repositories)
+    pub fn repo_type(mut self, repo_type: &str) -> Self {
+        if !repo_type.is_empty() {
+            self.config.repo_type = Some(repo_type.to_string());
+        }
+        self
+    }
+
+    /// Set repo id of this backend. This is required.
+    ///
+    /// Repo id consists of the account name and the repository name.
+    ///
+    /// For example, model's repo id looks like:
+    /// - meta-llama/Llama-2-7b
+    ///
+    /// Dataset's repo id looks like:
+    /// - databricks/databricks-dolly-15k
+    pub fn repo_id(mut self, repo_id: &str) -> Self {
+        if !repo_id.is_empty() {
+            self.config.repo_id = Some(repo_id.to_string());
+        }
+        self
+    }
+
+    /// Set revision of this backend. Default is main.
+    ///
+    /// Revision can be a branch name or a commit hash.
+    ///
+    /// For example, revision can be:
+    /// - main
+    /// - 1d0c4eb
+    pub fn revision(mut self, revision: &str) -> Self {
+        if !revision.is_empty() {
+            self.config.revision = Some(revision.to_string());
+        }
+        self
+    }
+
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set the token of this backend.
+    ///
+    /// This is optional.
+    pub fn token(mut self, token: &str) -> Self {
+        if !token.is_empty() {
+            self.config.token = Some(token.to_string());
+        }
+        self
+    }
+}
+
+impl Builder for HuggingfaceBuilder {
+    type Config = HuggingfaceConfig;
+
+    /// Build a HuggingfaceBackend.
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let repo_type = match self.config.repo_type.as_deref() {
+            Some("model") => Ok(RepoType::Model),
+            Some("dataset") => Ok(RepoType::Dataset),
+            Some("space") => Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                "repo type \"space\" is unsupported",
+            )),
+            Some(repo_type) => Err(Error::new(
+                ErrorKind::ConfigInvalid,
+                format!("unknown repo_type: {repo_type}").as_str(),
+            )
+            .with_operation("Builder::build")
+            .with_context("service", Scheme::Huggingface)),
+            None => Ok(RepoType::Model),
+        }?;
+        debug!("backend use repo_type: {:?}", &repo_type);
+
+        let repo_id = match &self.config.repo_id {
+            Some(repo_id) => Ok(repo_id.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "repo_id is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Huggingface)),
+        }?;
+        debug!("backend use repo_id: {}", &repo_id);
+
+        let revision = match &self.config.revision {
+            Some(revision) => revision.clone(),
+            None => "main".to_string(),
+        };
+        debug!("backend use revision: {}", &revision);
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root: {}", &root);
+
+        let token = self.config.token.as_ref().cloned();
+
+        Ok(HuggingfaceBackend {
+            core: Arc::new(HuggingfaceCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            read: true,
+
+                            list: true,
+                            list_with_recursive: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+                    am.into()
+                },
+                repo_type,
+                repo_id,
+                revision,
+                root,
+                token,
+            }),
+        })
+    }
+}
+
+/// Backend for Huggingface service
+#[derive(Debug, Clone)]
+pub struct HuggingfaceBackend {
+    core: Arc<HuggingfaceCore>,
+}
+
+impl Access for HuggingfaceBackend {
+    type Reader = HttpBody;
+    type Writer = ();
+    type Lister = oio::PageLister<HuggingfaceLister>;
+    type Deleter = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        // Stat root always returns a DIR.
+        if path == "/" {
+            return Ok(RpStat::new(Metadata::new(EntryMode::DIR)));
+        }
+
+        let resp = self.core.hf_path_info(path).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK => {
+                let mut meta = parse_into_metadata(path, resp.headers())?;
+                let bs = resp.into_body();
+
+                let decoded_response: Vec<HuggingfaceStatus> =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+                // NOTE: if the file is not found, the server will return 200 with an empty array
+                if let Some(status) = decoded_response.first() {
+                    if let Some(commit_info) = status.last_commit.as_ref() {
+                        meta.set_last_modified(parse_datetime_from_rfc3339(
+                            commit_info.date.as_str(),
+                        )?);
+                    }
+
+                    meta.set_content_length(status.size);
+
+                    match status.type_.as_str() {
+                        "directory" => meta.set_mode(EntryMode::DIR),
+                        "file" => meta.set_mode(EntryMode::FILE),
+                        _ => return Err(Error::new(ErrorKind::Unexpected, "unknown status type")),
+                    };
+                } else {
+                    return Err(Error::new(ErrorKind::NotFound, "path not found"));
+                }
+
+                Ok(RpStat::new(meta))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.hf_resolve(path, args.range(), &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
+                Ok((RpRead::default(), resp.into_body()))
+            }
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = HuggingfaceLister::new(self.core.clone(), path.to_string(), args.recursive());
+
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+}
+
+/// Repository type of Huggingface. Currently, we only support `model` and `dataset`.
+/// [Reference](https://huggingface.co/docs/hub/repositories)
+#[derive(Debug, Clone, Copy)]
+pub enum RepoType {
+    Model,
+    Dataset,
+}

data/core/src/services/huggingface/config.rs

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Configuration for Huggingface service support.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct HuggingfaceConfig {
+    /// Repo type of this backend. Default is model.
+    ///
+    /// Available values:
+    /// - model
+    /// - dataset
+    pub repo_type: Option<String>,
+    /// Repo id of this backend.
+    ///
+    /// This is required.
+    pub repo_id: Option<String>,
+    /// Revision of this backend.
+    ///
+    /// Default is main.
+    pub revision: Option<String>,
+    /// Root of this backend. Can be "/path/to/dir".
+    ///
+    /// Default is "/".
+    pub root: Option<String>,
+    /// Token of this backend.
+    ///
+    /// This is optional.
+    pub token: Option<String>,
+}
+
+impl Debug for HuggingfaceConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("HuggingfaceConfig");
+
+        if let Some(repo_type) = &self.repo_type {
+            ds.field("repo_type", &repo_type);
+        }
+        if let Some(repo_id) = &self.repo_id {
+            ds.field("repo_id", &repo_id);
+        }
+        if let Some(revision) = &self.revision {
+            ds.field("revision", &revision);
+        }
+        if let Some(root) = &self.root {
+            ds.field("root", &root);
+        }
+        if self.token.is_some() {
+            ds.field("token", &"<redacted>");
+        }
+
+        ds.finish()
+    }
+}