opendal 0.1.6.pre.rc.1

data/core/src/services/lakefs/core.rs

@@ -0,0 +1,261 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use http::header;
+use http::Request;
+use http::Response;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+pub struct LakefsCore {
+    pub info: Arc<AccessorInfo>,
+    pub endpoint: String,
+    pub repository: String,
+    pub branch: String,
+    pub root: String,
+    pub username: String,
+    pub password: String,
+}
+
+impl Debug for LakefsCore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LakefsCore")
+            .field("endpoint", &self.endpoint)
+            .field("username", &self.username)
+            .field("password", &self.password)
+            .field("root", &self.root)
+            .field("repository", &self.repository)
+            .field("branch", &self.branch)
+            .finish_non_exhaustive()
+    }
+}
+
+impl LakefsCore {
+    pub async fn get_object_metadata(&self, path: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/v1/repositories/{}/refs/{}/objects/stat?path={}",
+            self.endpoint,
+            self.repository,
+            self.branch,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        // Inject operation to the request.
+        let req = req.extension(Operation::Read);
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn get_object_content(
+        &self,
+        path: &str,
+        range: BytesRange,
+        _args: &OpRead,
+    ) -> Result<Response<HttpBody>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/v1/repositories/{}/refs/{}/objects?path={}",
+            self.endpoint,
+            self.repository,
+            self.branch,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+
+        if !range.is_full() {
+            req = req.header(header::RANGE, range.to_header());
+        }
+        // Inject operation to the request.
+        let req = req.extension(Operation::Read);
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().fetch(req).await
+    }
+
+    pub async fn list_objects(
+        &self,
+        path: &str,
+        delimiter: &str,
+        amount: &Option<usize>,
+        after: Option<String>,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path);
+
+        let mut url = format!(
+            "{}/api/v1/repositories/{}/refs/{}/objects/ls?",
+            self.endpoint, self.repository, self.branch
+        );
+
+        if !p.is_empty() {
+            url.push_str(&format!("&prefix={}", percent_encode_path(&p)));
+        }
+
+        if !delimiter.is_empty() {
+            url.push_str(&format!("&delimiter={delimiter}"));
+        }
+
+        if let Some(amount) = amount {
+            url.push_str(&format!("&amount={amount}"));
+        }
+
+        if let Some(after) = after {
+            url.push_str(&format!("&after={after}"));
+        }
+
+        let mut req = Request::get(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        // Inject operation to the request.
+        let req = req.extension(Operation::Read);
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn upload_object(
+        &self,
+        path: &str,
+        _args: &OpWrite,
+        body: Buffer,
+    ) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/v1/repositories/{}/branches/{}/objects?path={}",
+            self.endpoint,
+            self.repository,
+            self.branch,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        // Inject operation to the request.
+        let req = req.extension(Operation::Write);
+        let req = req.body(body).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn delete_object(&self, path: &str, _args: &OpDelete) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/v1/repositories/{}/branches/{}/objects?path={}",
+            self.endpoint,
+            self.repository,
+            self.branch,
+            percent_encode_path(&p)
+        );
+
+        let mut req = Request::delete(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        // Inject operation to the request.
+        let req = req.extension(Operation::Delete);
+        let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+
+        self.info.http_client().send(req).await
+    }
+
+    pub async fn copy_object(&self, path: &str, dest: &str) -> Result<Response<Buffer>> {
+        let p = build_abs_path(&self.root, path)
+            .trim_end_matches('/')
+            .to_string();
+        let d = build_abs_path(&self.root, dest)
+            .trim_end_matches('/')
+            .to_string();
+
+        let url = format!(
+            "{}/api/v1/repositories/{}/branches/{}/objects/copy?dest_path={}",
+            self.endpoint,
+            self.repository,
+            self.branch,
+            percent_encode_path(&d)
+        );
+
+        let mut req = Request::post(&url);
+
+        let auth_header_content = format_authorization_by_basic(&self.username, &self.password)?;
+        req = req.header(header::AUTHORIZATION, auth_header_content);
+        req = req.header(header::CONTENT_TYPE, "application/json");
+        let mut map = HashMap::new();
+        map.insert("src_path", p);
+
+        let req = req
+            // Inject operation to the request.
+            .extension(Operation::Delete)
+            .body(serde_json::to_vec(&map).unwrap().into())
+            .map_err(new_request_build_error)?;
+        self.info.http_client().send(req).await
+    }
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+pub(super) struct LakefsStatus {
+    pub path: String,
+    pub path_type: String,
+    pub physical_address: String,
+    pub checksum: String,
+    pub size_bytes: Option<u64>,
+    pub mtime: i64,
+    pub content_type: Option<String>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+pub(super) struct LakefsListResponse {
+    pub pagination: Pagination,
+    pub results: Vec<LakefsStatus>,
+}
+
+#[derive(Deserialize, Eq, PartialEq, Debug)]
+pub(super) struct Pagination {
+    pub has_more: bool,
+    pub max_per_page: u64,
+    pub next_offset: String,
+    pub results: u64,
+}

data/core/src/services/lakefs/delete.rs

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::*;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct LakefsDeleter {
+    core: Arc<LakefsCore>,
+}
+
+impl LakefsDeleter {
+    pub fn new(core: Arc<LakefsCore>) -> Self {
+        Self { core }
+    }
+}
+
+impl oio::OneShotDelete for LakefsDeleter {
+    async fn delete_once(&self, path: String, args: OpDelete) -> Result<()> {
+        // This would delete the bucket, do not perform
+        if self.core.root == "/" && path == "/" {
+            return Ok(());
+        }
+
+        let resp = self.core.delete_object(&path, &args).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::NO_CONTENT => Ok(()),
+            StatusCode::NOT_FOUND => Ok(()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}

data/core/src/services/lakefs/docs.md

@@ -0,0 +1,62 @@
+This service will visit the [Lakefs API](https://Lakefs.co/docs/Lakefs_hub/package_reference/hf_api) to access the Lakefs File System.
+Currently, we only support the `model` and `dataset` types of repositories, and operations are limited to reading and listing/stating.
+
+Lakefs doesn't host official HTTP API docs. Detailed HTTP request API information can be found on the [`Lakefs_hub` Source Code](https://github.com/Lakefs/Lakefs_hub).
+
+## Capabilities
+
+This service can be used to:
+
+- [x] stat
+- [x] read
+- [x] write
+- [ ] create_dir
+- [x] delete
+- [x] copy
+- [ ] rename
+- [x] list
+- [ ] ~~presign~~
+- [ ] blocking
+
+## Configurations
+
+- `endpoint`: The endpoint of the Lakefs repository.
+- `repository`: The id of the repository.
+- `branch`: The branch of the repository.
+- `root`: Set the work directory for backend.
+- `username`: The username for accessing the repository.
+- `password`: The password for accessing the repository.
+
+Refer to [`LakefsBuilder`]'s public API docs for more information.
+
+## Examples
+
+### Via Builder
+
+```rust,no_run
+use opendal::Operator;
+use opendal::services::Lakefs;
+use anyhow::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Create Lakefs backend builder
+    let mut builder = Lakefs::default()
+        // set the type of Lakefs endpoint
+        .endpoint("https://whole-llama-mh6mux.us-east-1.lakefscloud.io")
+        // set the id of Lakefs repository
+        .repository("sample-repo")
+        // set the branch of Lakefs repository
+        .branch("main")
+        // set the username for accessing the repository
+        .username("xxx")
+        // set the password for accessing the repository
+        .password("xxx");
+
+    let op: Operator = Operator::new(builder)?.finish();
+
+    let stat = op.stat("README.md").await?;
+    println!("{:?}", stat);
+    Ok(())
+}
+```

data/core/src/services/lakefs/error.rs

@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use http::Response;
+use http::StatusCode;
+use serde::Deserialize;
+
+use crate::raw::*;
+use crate::*;
+
+/// LakefsError is the error returned by Lakefs File System.
+#[derive(Default, Deserialize)]
+struct LakefsError {
+    error: String,
+}
+
+impl Debug for LakefsError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut de = f.debug_struct("LakefsError");
+        de.field("message", &self.error.replace('\n', " "));
+
+        de.finish()
+    }
+}
+
+pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
+    let (parts, body) = resp.into_parts();
+    let bs = body.to_bytes();
+
+    let (kind, retryable) = match parts.status {
+        StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
+        StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
+        StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
+        StatusCode::INTERNAL_SERVER_ERROR
+        | StatusCode::BAD_GATEWAY
+        | StatusCode::SERVICE_UNAVAILABLE
+        | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
+    let message = match serde_json::from_slice::<LakefsError>(&bs) {
+        Ok(hf_error) => format!("{:?}", hf_error.error),
+        Err(_) => String::from_utf8_lossy(&bs).into_owned(),
+    };
+
+    let mut err = Error::new(kind, message);
+
+    err = with_error_response_context(err, parts);
+
+    if retryable {
+        err = err.set_temporary();
+    }
+
+    err
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::raw::new_json_deserialize_error;
+    use crate::types::Result;
+
+    #[test]
+    fn test_parse_error() -> Result<()> {
+        let resp = r#"
+            {
+                "error": "Invalid username or password."
+            }
+            "#;
+        let decoded_response = serde_json::from_slice::<LakefsError>(resp.as_bytes())
+            .map_err(new_json_deserialize_error)?;
+
+        assert_eq!(decoded_response.error, "Invalid username or password.");
+
+        Ok(())
+    }
+}

data/core/src/services/lakefs/lister.rs

@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use chrono::TimeZone;
+use chrono::Utc;
+
+use super::core::LakefsCore;
+use super::core::LakefsListResponse;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct LakefsLister {
+    core: Arc<LakefsCore>,
+    path: String,
+    delimiter: &'static str,
+    amount: Option<usize>,
+    after: Option<String>,
+}
+
+impl LakefsLister {
+    pub fn new(
+        core: Arc<LakefsCore>,
+        path: String,
+        amount: Option<usize>,
+        after: Option<&str>,
+        recursive: bool,
+    ) -> Self {
+        let delimiter = if recursive { "" } else { "/" };
+        Self {
+            core,
+            path,
+            delimiter,
+            amount,
+            after: after.map(String::from),
+        }
+    }
+}
+
+impl oio::PageList for LakefsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let response = self
+            .core
+            .list_objects(
+                &self.path,
+                self.delimiter,
+                &self.amount,
+                // start after should only be set for the first page.
+                if ctx.token.is_empty() {
+                    self.after.clone()
+                } else {
+                    None
+                },
+            )
+            .await?;
+
+        let status_code = response.status();
+        if !status_code.is_success() {
+            let error = parse_error(response);
+            return Err(error);
+        }
+
+        let bytes = response.into_body();
+
+        let decoded_response: LakefsListResponse =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        ctx.done = true;
+
+        for status in decoded_response.results {
+            let entry_type = match status.path_type.as_str() {
+                "common_prefix" => EntryMode::DIR,
+                "object" => EntryMode::FILE,
+                _ => EntryMode::Unknown,
+            };
+
+            let mut meta = Metadata::new(entry_type);
+
+            if status.mtime != 0 {
+                meta.set_last_modified(Utc.timestamp_opt(status.mtime, 0).unwrap());
+            }
+
+            if entry_type == EntryMode::FILE {
+                if let Some(size_bytes) = status.size_bytes {
+                    meta.set_content_length(size_bytes);
+                }
+            }
+
+            let path = if entry_type == EntryMode::DIR {
+                format!("{}/", &status.path)
+            } else {
+                status.path.clone()
+            };
+
+            ctx.entries.push_back(oio::Entry::new(
+                &build_rel_path(&self.core.root, &path),
+                meta,
+            ));
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/lakefs/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for lakefs service.
+#[cfg(feature = "services-lakefs")]
+pub(super) const DEFAULT_SCHEME: &str = "lakefs";
+#[cfg(feature = "services-lakefs")]
+mod core;
+#[cfg(feature = "services-lakefs")]
+mod delete;
+#[cfg(feature = "services-lakefs")]
+mod error;
+#[cfg(feature = "services-lakefs")]
+mod lister;
+#[cfg(feature = "services-lakefs")]
+mod writer;
+
+#[cfg(feature = "services-lakefs")]
+mod backend;
+#[cfg(feature = "services-lakefs")]
+pub use backend::LakefsBuilder as Lakefs;
+
+mod config;
+pub use config::LakefsConfig;

data/core/src/services/lakefs/writer.rs

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::LakefsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct LakefsWriter {
+    core: Arc<LakefsCore>,
+    op: OpWrite,
+    path: String,
+}
+
+impl LakefsWriter {
+    pub fn new(core: Arc<LakefsCore>, path: String, op: OpWrite) -> Self {
+        LakefsWriter { core, path, op }
+    }
+}
+
+impl oio::OneShotWrite for LakefsWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let resp = self.core.upload_object(&self.path, &self.op, bs).await?;
+
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}