opendal 0.1.6.pre.rc.1

data/core/src/services/azdls/lister.rs

@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use serde::Deserialize;
+use serde_json::de;
+
+use super::core::AzdlsCore;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub struct AzdlsLister {
+    core: Arc<AzdlsCore>,
+
+    path: String,
+    limit: Option<usize>,
+}
+
+impl AzdlsLister {
+    pub fn new(core: Arc<AzdlsCore>, path: String, limit: Option<usize>) -> Self {
+        Self { core, path, limit }
+    }
+}
+
+impl oio::PageList for AzdlsLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let resp = self
+            .core
+            .azdls_list(&self.path, &ctx.token, self.limit)
+            .await?;
+
+        // azdls will return not found for not-exist path.
+        if resp.status() == http::StatusCode::NOT_FOUND {
+            ctx.done = true;
+            return Ok(());
+        }
+
+        if resp.status() != http::StatusCode::OK {
+            return Err(parse_error(resp));
+        }
+
+        // Return self at the first page.
+        if ctx.token.is_empty() && !ctx.done {
+            let e = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR));
+            ctx.entries.push_back(e);
+        }
+
+        // Check whether this list is done.
+        if let Some(value) = resp.headers().get("x-ms-continuation") {
+            let value = value.to_str().map_err(|err| {
+                Error::new(ErrorKind::Unexpected, "header value is not valid string")
+                    .set_source(err)
+            })?;
+            ctx.token = value.to_string();
+        } else {
+            ctx.token = "".to_string();
+            ctx.done = true;
+        }
+
+        let bs = resp.into_body();
+
+        let output: Output = de::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+
+        for object in output.paths {
+            // Azdls will return `"true"` and `"false"` for is_directory.
+            let mode = if &object.is_directory == "true" {
+                EntryMode::DIR
+            } else {
+                EntryMode::FILE
+            };
+
+            let meta = Metadata::new(mode)
+                // Keep fit with ETag header.
+                .with_etag(format!("\"{}\"", &object.etag))
+                .with_content_length(object.content_length.parse().map_err(|err| {
+                    Error::new(ErrorKind::Unexpected, "content length is not valid integer")
+                        .set_source(err)
+                })?)
+                .with_last_modified(parse_datetime_from_rfc2822(&object.last_modified)?);
+
+            let mut path = build_rel_path(&self.core.root, &object.name);
+            if mode.is_dir() {
+                path += "/"
+            };
+
+            let de = oio::Entry::new(&path, meta);
+
+            ctx.entries.push_back(de);
+        }
+
+        Ok(())
+    }
+}
+
+/// # Examples
+///
+/// ```json
+/// {"paths":[{"contentLength":"1977097","etag":"0x8DACF9B0061305F","group":"$superuser","lastModified":"Sat, 26 Nov 2022 10:43:05 GMT","name":"c3b3ef48-7783-4946-81bc-dc07e1728878/d4ea21d7-a533-4011-8b1f-d0e566d63725","owner":"$superuser","permissions":"rw-r-----"}]}
+/// ```
+#[derive(Default, Debug, Deserialize)]
+#[serde(default)]
+struct Output {
+    paths: Vec<Path>,
+}
+
+#[derive(Default, Debug, Deserialize, PartialEq, Eq)]
+#[serde(default)]
+struct Path {
+    #[serde(rename = "contentLength")]
+    content_length: String,
+    #[serde(rename = "etag")]
+    etag: String,
+    /// Azdls will return `"true"` and `"false"` for is_directory.
+    #[serde(rename = "isDirectory")]
+    is_directory: String,
+    #[serde(rename = "lastModified")]
+    last_modified: String,
+    #[serde(rename = "name")]
+    name: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Bytes;
+
+    use super::*;
+
+    #[test]
+    fn test_parse_path() {
+        let bs = Bytes::from(
+            r#"{"paths":[{"contentLength":"1977097","etag":"0x8DACF9B0061305F","group":"$superuser","lastModified":"Sat, 26 Nov 2022 10:43:05 GMT","name":"c3b3ef48-7783-4946-81bc-dc07e1728878/d4ea21d7-a533-4011-8b1f-d0e566d63725","owner":"$superuser","permissions":"rw-r-----"}]}"#,
+        );
+        let out: Output = de::from_slice(&bs).expect("must success");
+        println!("{out:?}");
+
+        assert_eq!(
+            out.paths[0],
+            Path {
+                content_length: "1977097".to_string(),
+                etag: "0x8DACF9B0061305F".to_string(),
+                is_directory: "".to_string(),
+                last_modified: "Sat, 26 Nov 2022 10:43:05 GMT".to_string(),
+                name: "c3b3ef48-7783-4946-81bc-dc07e1728878/d4ea21d7-a533-4011-8b1f-d0e566d63725"
+                    .to_string()
+            }
+        );
+    }
+}

data/core/src/services/azdls/mod.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for azdls service.
+#[cfg(feature = "services-azdls")]
+pub(super) const DEFAULT_SCHEME: &str = "azdls";
+#[cfg(feature = "services-azdls")]
+mod backend;
+#[cfg(feature = "services-azdls")]
+mod core;
+#[cfg(feature = "services-azdls")]
+mod delete;
+#[cfg(feature = "services-azdls")]
+mod error;
+#[cfg(feature = "services-azdls")]
+mod lister;
+#[cfg(feature = "services-azdls")]
+mod writer;
+
+#[cfg(feature = "services-azdls")]
+pub use backend::AzdlsBuilder as Azdls;
+
+mod config;
+pub use config::AzdlsConfig;

data/core/src/services/azdls/writer.rs

@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use http::StatusCode;
+
+use super::core::AzdlsCore;
+use super::core::FILE;
+use super::core::X_MS_VERSION_ID;
+use super::error::parse_error;
+use crate::raw::*;
+use crate::*;
+
+pub type AzdlsWriters = TwoWays<oio::OneShotWriter<AzdlsWriter>, oio::AppendWriter<AzdlsWriter>>;
+
+pub struct AzdlsWriter {
+    core: Arc<AzdlsCore>,
+
+    op: OpWrite,
+    path: String,
+}
+
+impl AzdlsWriter {
+    pub fn new(core: Arc<AzdlsCore>, op: OpWrite, path: String) -> Self {
+        AzdlsWriter { core, op, path }
+    }
+
+    fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
+        let mut metadata = Metadata::default();
+
+        if let Some(last_modified) = parse_last_modified(headers)? {
+            metadata.set_last_modified(last_modified);
+        }
+        let etag = parse_etag(headers)?;
+        if let Some(etag) = etag {
+            metadata.set_etag(etag);
+        }
+        let version_id = parse_header_to_str(headers, X_MS_VERSION_ID)?;
+        if let Some(version_id) = version_id {
+            metadata.set_version(version_id);
+        }
+
+        Ok(metadata)
+    }
+}
+
+impl oio::OneShotWrite for AzdlsWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let resp = self.core.azdls_create(&self.path, FILE, &self.op).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::CREATED | StatusCode::OK => {}
+            _ => {
+                return Err(parse_error(resp).with_operation("Backend::azdls_create_request"));
+            }
+        }
+
+        let resp = self
+            .core
+            .azdls_update(&self.path, Some(bs.len() as u64), 0, bs)
+            .await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::ACCEPTED => Ok(Metadata::default()),
+            _ => Err(parse_error(resp).with_operation("Backend::azdls_update_request")),
+        }
+    }
+}
+
+impl oio::AppendWrite for AzdlsWriter {
+    async fn offset(&self) -> Result<u64> {
+        let resp = self.core.azdls_get_properties(&self.path).await?;
+
+        let status = resp.status();
+        let headers = resp.headers();
+
+        match status {
+            StatusCode::OK => Ok(parse_content_length(headers)?.unwrap_or_default()),
+            StatusCode::NOT_FOUND => Ok(0),
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn append(&self, offset: u64, size: u64, body: Buffer) -> Result<Metadata> {
+        if offset == 0 {
+            let resp = self.core.azdls_create(&self.path, FILE, &self.op).await?;
+            let status = resp.status();
+            match status {
+                StatusCode::CREATED | StatusCode::OK => {}
+                _ => {
+                    return Err(parse_error(resp).with_operation("Backend::azdls_create_request"));
+                }
+            }
+        }
+
+        let resp = self
+            .core
+            .azdls_update(&self.path, Some(size), offset, body)
+            .await?;
+
+        let mut meta = AzdlsWriter::parse_metadata(resp.headers())?;
+        let md5 = parse_content_md5(resp.headers())?;
+        if let Some(md5) = md5 {
+            meta.set_content_md5(md5);
+        }
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::ACCEPTED => Ok(meta),
+            _ => Err(parse_error(resp).with_operation("Backend::azdls_update_request")),
+        }
+    }
+}

data/core/src/services/azfile/backend.rs

@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Formatter;
+use std::sync::Arc;
+
+use http::Response;
+use http::StatusCode;
+use log::debug;
+use reqsign::AzureStorageConfig;
+use reqsign::AzureStorageLoader;
+use reqsign::AzureStorageSigner;
+
+use super::core::AzfileCore;
+use super::delete::AzfileDeleter;
+use super::error::parse_error;
+use super::lister::AzfileLister;
+use super::writer::AzfileWriter;
+use super::writer::AzfileWriters;
+use super::DEFAULT_SCHEME;
+use crate::raw::*;
+use crate::services::AzfileConfig;
+use crate::*;
+impl From<AzureStorageConfig> for AzfileConfig {
+    fn from(config: AzureStorageConfig) -> Self {
+        AzfileConfig {
+            account_name: config.account_name,
+            account_key: config.account_key,
+            sas_token: config.sas_token,
+            endpoint: config.endpoint,
+            root: None,                // root is not part of AzureStorageConfig
+            share_name: String::new(), // share_name is not part of AzureStorageConfig
+        }
+    }
+}
+
+impl Configurator for AzfileConfig {
+    type Builder = AzfileBuilder;
+
+    #[allow(deprecated)]
+    fn into_builder(self) -> Self::Builder {
+        AzfileBuilder {
+            config: self,
+            http_client: None,
+        }
+    }
+}
+
+/// Azure File services support.
+#[doc = include_str!("docs.md")]
+#[derive(Default, Clone)]
+pub struct AzfileBuilder {
+    config: AzfileConfig,
+
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    http_client: Option<HttpClient>,
+}
+
+impl Debug for AzfileBuilder {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("AzfileBuilder");
+
+        ds.field("config", &self.config);
+
+        ds.finish()
+    }
+}
+
+impl AzfileBuilder {
+    /// Set root of this backend.
+    ///
+    /// All operations will happen under this root.
+    pub fn root(mut self, root: &str) -> Self {
+        self.config.root = if root.is_empty() {
+            None
+        } else {
+            Some(root.to_string())
+        };
+
+        self
+    }
+
+    /// Set endpoint of this backend.
+    pub fn endpoint(mut self, endpoint: &str) -> Self {
+        if !endpoint.is_empty() {
+            // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
+            self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
+        }
+
+        self
+    }
+
+    /// Set account_name of this backend.
+    ///
+    /// - If account_name is set, we will take user's input first.
+    /// - If not, we will try to load it from environment.
+    pub fn account_name(mut self, account_name: &str) -> Self {
+        if !account_name.is_empty() {
+            self.config.account_name = Some(account_name.to_string());
+        }
+
+        self
+    }
+
+    /// Set account_key of this backend.
+    ///
+    /// - If account_key is set, we will take user's input first.
+    /// - If not, we will try to load it from environment.
+    pub fn account_key(mut self, account_key: &str) -> Self {
+        if !account_key.is_empty() {
+            self.config.account_key = Some(account_key.to_string());
+        }
+
+        self
+    }
+
+    /// Set file share name of this backend.
+    ///
+    /// # Notes
+    /// You can find more about from: <https://learn.microsoft.com/en-us/rest/api/storageservices/operations-on-shares--file-service>
+    pub fn share_name(mut self, share_name: &str) -> Self {
+        if !share_name.is_empty() {
+            self.config.share_name = share_name.to_string();
+        }
+
+        self
+    }
+
+    /// Specify the http client that used by this service.
+    ///
+    /// # Notes
+    ///
+    /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed
+    /// during minor updates.
+    #[deprecated(since = "0.53.0", note = "Use `Operator::update_http_client` instead")]
+    #[allow(deprecated)]
+    pub fn http_client(mut self, client: HttpClient) -> Self {
+        self.http_client = Some(client);
+        self
+    }
+
+    /// Create a new `AfileBuilder` instance from an [Azure Storage connection string][1].
+    ///
+    /// [1]: https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string
+    ///
+    /// # Example
+    /// ```
+    /// use opendal::Builder;
+    /// use opendal::services::Azfile;
+    ///
+    /// let conn_str = "AccountName=example;DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net";
+    ///
+    /// let mut config = Azfile::from_connection_string(&conn_str)
+    ///     .unwrap()
+    ///     // Add additional configuration if needed
+    ///     .share_name("myShare")
+    ///     .build()
+    ///     .unwrap();
+    /// ```
+    pub fn from_connection_string(conn_str: &str) -> Result<Self> {
+        let config =
+            raw::azure_config_from_connection_string(conn_str, raw::AzureStorageService::File)?;
+
+        Ok(AzfileConfig::from(config).into_builder())
+    }
+}
+
+impl Builder for AzfileBuilder {
+    type Config = AzfileConfig;
+
+    fn build(self) -> Result<impl Access> {
+        debug!("backend build started: {:?}", &self);
+
+        let root = normalize_root(&self.config.root.unwrap_or_default());
+        debug!("backend use root {root}");
+
+        let endpoint = match &self.config.endpoint {
+            Some(endpoint) => Ok(endpoint.clone()),
+            None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
+                .with_operation("Builder::build")
+                .with_context("service", Scheme::Azfile)),
+        }?;
+        debug!("backend use endpoint {}", &endpoint);
+
+        let account_name_option = self
+            .config
+            .account_name
+            .clone()
+            .or_else(|| raw::azure_account_name_from_endpoint(endpoint.as_str()));
+
+        let account_name = match account_name_option {
+            Some(account_name) => Ok(account_name),
+            None => Err(
+                Error::new(ErrorKind::ConfigInvalid, "account_name is empty")
+                    .with_operation("Builder::build")
+                    .with_context("service", Scheme::Azfile),
+            ),
+        }?;
+
+        let config_loader = AzureStorageConfig {
+            account_name: Some(account_name),
+            account_key: self.config.account_key.clone(),
+            sas_token: self.config.sas_token.clone(),
+            ..Default::default()
+        };
+
+        let cred_loader = AzureStorageLoader::new(config_loader);
+        let signer = AzureStorageSigner::new();
+        Ok(AzfileBackend {
+            core: Arc::new(AzfileCore {
+                info: {
+                    let am = AccessorInfo::default();
+                    am.set_scheme(DEFAULT_SCHEME)
+                        .set_root(&root)
+                        .set_native_capability(Capability {
+                            stat: true,
+
+                            read: true,
+
+                            write: true,
+                            create_dir: true,
+                            delete: true,
+                            rename: true,
+
+                            list: true,
+
+                            shared: true,
+
+                            ..Default::default()
+                        });
+
+                    // allow deprecated api here for compatibility
+                    #[allow(deprecated)]
+                    if let Some(client) = self.http_client {
+                        am.update_http_client(|_| client);
+                    }
+
+                    am.into()
+                },
+                root,
+                endpoint,
+                loader: cred_loader,
+                signer,
+                share_name: self.config.share_name.clone(),
+            }),
+        })
+    }
+}
+
+/// Backend for azfile services.
+#[derive(Debug, Clone)]
+pub struct AzfileBackend {
+    core: Arc<AzfileCore>,
+}
+
+impl Access for AzfileBackend {
+    type Reader = HttpBody;
+    type Writer = AzfileWriters;
+    type Lister = oio::PageLister<AzfileLister>;
+    type Deleter = oio::OneShotDeleter<AzfileDeleter>;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.core.info.clone()
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        self.core.ensure_parent_dir_exists(path).await?;
+        let resp = self.core.azfile_create_dir(path).await?;
+        let status = resp.status();
+
+        match status {
+            StatusCode::CREATED => Ok(RpCreateDir::default()),
+            _ => {
+                // we cannot just check status code because 409 Conflict has two meaning:
+                // 1. If a directory by the same name is being deleted when Create Directory is called, the server returns status code 409 (Conflict)
+                // 2. If a directory or file with the same name already exists, the operation fails with status code 409 (Conflict).
+                // but we just need case 2 (already exists)
+                // ref: https://learn.microsoft.com/en-us/rest/api/storageservices/create-directory
+                if resp
+                    .headers()
+                    .get("x-ms-error-code")
+                    .map(|value| value.to_str().unwrap_or(""))
+                    .unwrap_or_else(|| "")
+                    == "ResourceAlreadyExists"
+                {
+                    Ok(RpCreateDir::default())
+                } else {
+                    Err(parse_error(resp))
+                }
+            }
+        }
+    }
+
+    async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
+        let resp = if path.ends_with('/') {
+            self.core.azfile_get_directory_properties(path).await?
+        } else {
+            self.core.azfile_get_file_properties(path).await?
+        };
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK => {
+                let meta = parse_into_metadata(path, resp.headers())?;
+                Ok(RpStat::new(meta))
+            }
+            _ => Err(parse_error(resp)),
+        }
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        let resp = self.core.azfile_read(path, args.range()).await?;
+
+        let status = resp.status();
+        match status {
+            StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok((RpRead::new(), resp.into_body())),
+            _ => {
+                let (part, mut body) = resp.into_parts();
+                let buf = body.to_buffer().await?;
+                Err(parse_error(Response::from_parts(part, buf)))
+            }
+        }
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        self.core.ensure_parent_dir_exists(path).await?;
+        let w = AzfileWriter::new(self.core.clone(), args.clone(), path.to_string());
+        let w = if args.append() {
+            AzfileWriters::Two(oio::AppendWriter::new(w))
+        } else {
+            AzfileWriters::One(oio::OneShotWriter::new(w))
+        };
+        Ok((RpWrite::default(), w))
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        Ok((
+            RpDelete::default(),
+            oio::OneShotDeleter::new(AzfileDeleter::new(self.core.clone())),
+        ))
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        let l = AzfileLister::new(self.core.clone(), path.to_string(), args.limit());
+
+        Ok((RpList::default(), oio::PageLister::new(l)))
+    }
+
+    async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result<RpRename> {
+        self.core.ensure_parent_dir_exists(to).await?;
+        let resp = self.core.azfile_rename(from, to).await?;
+        let status = resp.status();
+        match status {
+            StatusCode::OK => Ok(RpRename::default()),
+            _ => Err(parse_error(resp)),
+        }
+    }
+}