opendal 0.1.6.pre.rc.1

data/core/src/raw/http_util/uri.rs

@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use percent_encoding::percent_decode_str;
+use percent_encoding::utf8_percent_encode;
+use percent_encoding::AsciiSet;
+use percent_encoding::NON_ALPHANUMERIC;
+
+use crate::*;
+
+/// Parse http uri invalid error in to opendal::Error.
+pub fn new_http_uri_invalid_error(err: http::uri::InvalidUri) -> Error {
+    Error::new(ErrorKind::Unexpected, "parse http uri").set_source(err)
+}
+
+/// PATH_ENCODE_SET is the encode set for http url path.
+///
+/// This set follows [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) which will encode all non-ASCII characters except `A-Z a-z 0-9 - _ . ! ~ * ' ( )`
+///
+/// There is a special case for `/` in path: we will allow `/` in path as
+/// required by storage services like s3.
+static PATH_ENCODE_SET: AsciiSet = NON_ALPHANUMERIC
+    .remove(b'/')
+    .remove(b'-')
+    .remove(b'_')
+    .remove(b'.')
+    .remove(b'!')
+    .remove(b'~')
+    .remove(b'*')
+    .remove(b'\'')
+    .remove(b'(')
+    .remove(b')');
+
+/// percent_encode_path will do percent encoding for http encode path.
+///
+/// Follows [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) which will encode all non-ASCII characters except `A-Z a-z 0-9 - _ . ! ~ * ' ( )`
+///
+/// There is a special case for `/` in path: we will allow `/` in path as
+/// required by storage services like s3.
+pub fn percent_encode_path(path: &str) -> String {
+    utf8_percent_encode(path, &PATH_ENCODE_SET).to_string()
+}
+
+/// percent_decode_path will do percent decoding for http decode path.
+///
+/// If the input is not percent encoded or not valid utf8, return the input.
+pub fn percent_decode_path(path: &str) -> String {
+    match percent_decode_str(path).decode_utf8() {
+        Ok(v) => v.to_string(),
+        Err(_) => path.to_string(),
+    }
+}
+
+/// QueryPairsWriter is used to write query pairs to a url.
+pub struct QueryPairsWriter {
+    base: String,
+    has_query: bool,
+}
+
+impl QueryPairsWriter {
+    /// Create a new QueryPairsWriter with the given base.
+    pub fn new(s: &str) -> Self {
+        // 256 is the average size we observed of a url
+        // in production.
+        //
+        // We eagerly allocate the string to avoid multiple
+        // allocations.
+        let mut base = String::with_capacity(256);
+        base.push_str(s);
+
+        Self {
+            base,
+            has_query: false,
+        }
+    }
+
+    /// Push a new pair of key and value to the url.
+    ///
+    /// The input key and value must already been percent
+    /// encoded correctly.
+    pub fn push(mut self, key: &str, value: &str) -> Self {
+        if self.has_query {
+            self.base.push('&');
+        } else {
+            self.base.push('?');
+            self.has_query = true;
+        }
+
+        // Append the key and value to the base string
+        self.base.push_str(key);
+        if !value.is_empty() {
+            self.base.push('=');
+            self.base.push_str(value);
+        }
+
+        self
+    }
+
+    /// Finish the url and return it.
+    pub fn finish(self) -> String {
+        self.base
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_percent_encode_path() {
+        let cases = vec![
+            (
+                "Reserved Characters",
+                ";,/?:@&=+$",
+                "%3B%2C/%3F%3A%40%26%3D%2B%24",
+            ),
+            ("Unescaped Characters", "-_.!~*'()", "-_.!~*'()"),
+            ("Number Sign", "#", "%23"),
+            (
+                "Alphanumeric Characters + Space",
+                "ABC abc 123",
+                "ABC%20abc%20123",
+            ),
+            (
+                "Unicode",
+                "你好，世界！❤",
+                "%E4%BD%A0%E5%A5%BD%EF%BC%8C%E4%B8%96%E7%95%8C%EF%BC%81%E2%9D%A4",
+            ),
+        ];
+
+        for (name, input, expected) in cases {
+            let actual = percent_encode_path(input);
+
+            assert_eq!(actual, expected, "{name}");
+        }
+    }
+
+    #[test]
+    fn test_percent_decode_path() {
+        let cases = vec![
+            (
+                "Reserved Characters",
+                "%3B%2C/%3F%3A%40%26%3D%2B%24",
+                ";,/?:@&=+$",
+            ),
+            ("Unescaped Characters", "-_.!~*'()", "-_.!~*'()"),
+            ("Number Sign", "%23", "#"),
+            (
+                "Alphanumeric Characters + Space",
+                "ABC%20abc%20123",
+                "ABC abc 123",
+            ),
+            (
+                "Unicode Characters",
+                "%E4%BD%A0%E5%A5%BD%EF%BC%8C%E4%B8%96%E7%95%8C%EF%BC%81%E2%9D%A4",
+                "你好，世界！❤",
+            ),
+            (
+                "Double Encoded Characters",
+                "Double%2520Encoded",
+                "Double%20Encoded",
+            ),
+            (
+                "Not Percent Encoded Characters",
+                "/not percent encoded/path;,/?:@&=+$-",
+                "/not percent encoded/path;,/?:@&=+$-",
+            ),
+        ];
+
+        for (name, input, expected) in cases {
+            let actual = percent_decode_path(input);
+
+            assert_eq!(actual, expected, "{name}");
+        }
+    }
+}

data/core/src/raw/layer.rs

@@ -0,0 +1,295 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+
+use futures::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// Layer is used to intercept the operations on the underlying storage.
+///
+/// Struct that implement this trait must accept input `Arc<dyn Accessor>` as inner,
+/// and returns a new `Arc<dyn Accessor>` as output.
+///
+/// All functions in `Accessor` requires `&self`, so it's implementer's responsibility
+/// to maintain the internal mutability. Please also keep in mind that `Accessor`
+/// requires `Send` and `Sync`.
+///
+/// # Notes
+///
+/// ## Inner
+///
+/// It's required to implement `fn inner() -> Option<Arc<dyn Accessor>>` for layer's accessors.
+///
+/// By implement this method, all API calls will be forwarded to inner accessor instead.
+///
+/// # Examples
+///
+/// ```
+/// use std::sync::Arc;
+///
+/// use opendal::raw::*;
+/// use opendal::*;
+///
+/// /// Implement the real accessor logic here.
+/// #[derive(Debug)]
+/// struct TraceAccessor<A: Access> {
+///     inner: A,
+/// }
+///
+/// impl<A: Access> LayeredAccess for TraceAccessor<A> {
+///     type Inner = A;
+///     type Reader = A::Reader;
+///     type Writer = A::Writer;
+///     type Lister = A::Lister;
+///     type Deleter = A::Deleter;
+///
+///     fn inner(&self) -> &Self::Inner {
+///         &self.inner
+///     }
+///
+///     async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+///         self.inner.read(path, args).await
+///     }
+///
+///     async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+///         self.inner.write(path, args).await
+///     }
+///
+///     async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+///         self.inner.list(path, args).await
+///     }
+///
+///     async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+///         self.inner.delete().await
+///     }
+/// }
+///
+/// /// The public struct that exposed to users.
+/// ///
+/// /// Will be used like `op.layer(TraceLayer)`
+/// struct TraceLayer;
+///
+/// impl<A: Access> Layer<A> for TraceLayer {
+///     type LayeredAccess = TraceAccessor<A>;
+///
+///     fn layer(&self, inner: A) -> Self::LayeredAccess {
+///         TraceAccessor { inner }
+///     }
+/// }
+/// ```
+pub trait Layer<A: Access> {
+    /// The layered accessor that returned by this layer.
+    type LayeredAccess: Access;
+
+    /// Intercept the operations on the underlying storage.
+    fn layer(&self, inner: A) -> Self::LayeredAccess;
+}
+
+/// LayeredAccess is layered accessor that forward all not implemented
+/// method to inner.
+#[allow(missing_docs)]
+pub trait LayeredAccess: Send + Sync + Debug + Unpin + 'static {
+    type Inner: Access;
+
+    type Reader: oio::Read;
+    type Writer: oio::Write;
+    type Lister: oio::List;
+    type Deleter: oio::Delete;
+
+    fn inner(&self) -> &Self::Inner;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        self.inner().info()
+    }
+
+    fn create_dir(
+        &self,
+        path: &str,
+        args: OpCreateDir,
+    ) -> impl Future<Output = Result<RpCreateDir>> + MaybeSend {
+        self.inner().create_dir(path, args)
+    }
+
+    fn read(
+        &self,
+        path: &str,
+        args: OpRead,
+    ) -> impl Future<Output = Result<(RpRead, Self::Reader)>> + MaybeSend;
+
+    fn write(
+        &self,
+        path: &str,
+        args: OpWrite,
+    ) -> impl Future<Output = Result<(RpWrite, Self::Writer)>> + MaybeSend;
+
+    fn copy(
+        &self,
+        from: &str,
+        to: &str,
+        args: OpCopy,
+    ) -> impl Future<Output = Result<RpCopy>> + MaybeSend {
+        self.inner().copy(from, to, args)
+    }
+
+    fn rename(
+        &self,
+        from: &str,
+        to: &str,
+        args: OpRename,
+    ) -> impl Future<Output = Result<RpRename>> + MaybeSend {
+        self.inner().rename(from, to, args)
+    }
+
+    fn stat(&self, path: &str, args: OpStat) -> impl Future<Output = Result<RpStat>> + MaybeSend {
+        self.inner().stat(path, args)
+    }
+
+    fn delete(&self) -> impl Future<Output = Result<(RpDelete, Self::Deleter)>> + MaybeSend;
+
+    fn list(
+        &self,
+        path: &str,
+        args: OpList,
+    ) -> impl Future<Output = Result<(RpList, Self::Lister)>> + MaybeSend;
+
+    fn presign(
+        &self,
+        path: &str,
+        args: OpPresign,
+    ) -> impl Future<Output = Result<RpPresign>> + MaybeSend {
+        self.inner().presign(path, args)
+    }
+}
+
+impl<L: LayeredAccess> Access for L {
+    type Reader = L::Reader;
+    type Writer = L::Writer;
+    type Lister = L::Lister;
+    type Deleter = L::Deleter;
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        LayeredAccess::info(self)
+    }
+
+    async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
+        LayeredAccess::create_dir(self, path, args).await
+    }
+
+    async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
+        LayeredAccess::read(self, path, args).await
+    }
+
+    async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
+        LayeredAccess::write(self, path, args).await
+    }
+
+    async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
+        LayeredAccess::copy(self, from, to, args).await
+    }
+
+    async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
+        LayeredAccess::rename(self, from, to, args).await
+    }
+
+    async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
+        LayeredAccess::stat(self, path, args).await
+    }
+
+    async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
+        LayeredAccess::delete(self).await
+    }
+
+    async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
+        LayeredAccess::list(self, path, args).await
+    }
+
+    async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
+        LayeredAccess::presign(self, path, args).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use futures::lock::Mutex;
+
+    use super::*;
+    use crate::services::Memory;
+
+    #[derive(Debug)]
+    struct Test<A: Access> {
+        #[allow(dead_code)]
+        inner: Option<A>,
+        stated: Arc<Mutex<bool>>,
+    }
+
+    impl<A: Access> Layer<A> for &Test<A> {
+        type LayeredAccess = Test<A>;
+
+        fn layer(&self, inner: A) -> Self::LayeredAccess {
+            Test {
+                inner: Some(inner),
+                stated: self.stated.clone(),
+            }
+        }
+    }
+
+    impl<A: Access> Access for Test<A> {
+        type Reader = ();
+        type Writer = ();
+        type Lister = ();
+        type Deleter = ();
+
+        fn info(&self) -> Arc<AccessorInfo> {
+            let am = AccessorInfo::default();
+            am.set_scheme("test");
+            am.into()
+        }
+
+        async fn stat(&self, _: &str, _: OpStat) -> Result<RpStat> {
+            let mut x = self.stated.lock().await;
+            *x = true;
+
+            assert!(self.inner.is_some());
+
+            // We will not call anything here to test the layer.
+            Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_layer() {
+        let test = Test {
+            inner: None,
+            stated: Arc::new(Mutex::new(false)),
+        };
+
+        let op = Operator::new(Memory::default())
+            .unwrap()
+            .layer(&test)
+            .finish();
+
+        op.stat("xxxxx").await.unwrap();
+
+        assert!(*test.stated.clone().lock().await);
+    }
+}

data/core/src/raw/mod.rs

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Raw modules provide raw APIs that used by underlying services
+//!
+//! ## Notes
+//!
+//! - Only developers who want to develop new services or layers need to
+//!   access raw APIs.
+//! - Raw APIs should only be accessed via `opendal::raw::Xxxx`, any public
+//!   API should never expose raw API directly.
+//! - Raw APIs are far less stable than public API, please don't rely on
+//!   them whenever possible.
+
+mod accessor;
+pub use accessor::*;
+
+#[cfg(any(
+    feature = "services-azblob",
+    feature = "services-azdls",
+    feature = "services-azfile"
+))]
+mod azure;
+#[cfg(any(
+    feature = "services-azblob",
+    feature = "services-azdls",
+    feature = "services-azfile"
+))]
+pub(crate) use azure::*;
+
+mod layer;
+pub use layer::*;
+
+mod path;
+pub use path::*;
+
+#[cfg(feature = "internal-path-cache")]
+mod path_cache;
+#[cfg(feature = "internal-path-cache")]
+pub use path_cache::*;
+
+mod operation;
+pub use operation::*;
+
+mod version;
+pub use version::VERSION;
+
+mod rps;
+pub use rps::*;
+
+mod ops;
+pub use ops::*;
+
+mod http_util;
+pub use http_util::*;
+
+mod serde_util;
+pub use serde_util::*;
+
+mod chrono_util;
+pub use chrono_util::*;
+
+#[cfg(feature = "internal-tokio-rt")]
+mod tokio_util;
+#[cfg(feature = "internal-tokio-rt")]
+pub use tokio_util::*;
+
+mod std_io_util;
+pub use std_io_util::*;
+
+mod futures_util;
+pub use futures_util::BoxedFuture;
+pub use futures_util::BoxedStaticFuture;
+pub use futures_util::ConcurrentTasks;
+pub use futures_util::MaybeSend;
+
+mod enum_utils;
+pub use enum_utils::*;
+
+mod atomic_util;
+pub use atomic_util::*;
+
+// Expose as a pub mod to avoid confusing.
+pub mod adapters;
+pub mod oio;
+#[cfg(feature = "tests")]
+pub mod tests;

data/core/src/raw/oio/buf/flex_buf.rs

@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use bytes::Buf;
+use bytes::BufMut;
+use bytes::Bytes;
+use bytes::BytesMut;
+
+/// FlexBuf is a buffer that support frozen bytes and reuse existing allocated memory.
+///
+/// It's useful when we want to freeze the buffer and reuse the memory for the next buffer.
+pub struct FlexBuf {
+    /// Already allocated memory size of `buf`.
+    cap: usize,
+    /// Already written bytes length inside `buf`.
+    len: usize,
+    buf: BytesMut,
+    frozen: Option<Bytes>,
+}
+
+impl FlexBuf {
+    /// Initializes a new `FlexBuf` with the given capacity.
+    pub fn new(cap: usize) -> Self {
+        FlexBuf {
+            cap,
+            len: 0,
+
+            buf: BytesMut::with_capacity(cap),
+            frozen: None,
+        }
+    }
+
+    /// Put slice into flex buf.
+    ///
+    /// Return 0 means the buffer is frozen.
+    pub fn put(&mut self, bs: &[u8]) -> usize {
+        if self.frozen.is_some() {
+            return 0;
+        }
+
+        let n = (self.cap - self.len).min(bs.len());
+        self.buf.put_slice(&bs[..n]);
+        self.len += n;
+
+        if self.len >= self.cap {
+            let frozen = self.buf.split();
+            self.len = 0;
+            self.frozen = Some(frozen.freeze());
+        }
+
+        n
+    }
+
+    /// Freeze the buffer no matter it's full or not.
+    ///
+    /// It's a no-op if the buffer has already been frozen.
+    pub fn freeze(&mut self) {
+        if self.len == 0 {
+            return;
+        }
+        let frozen = self.buf.split();
+        self.len = 0;
+        self.frozen = Some(frozen.freeze());
+    }
+
+    /// Get the frozen buffer.
+    ///
+    /// Return `None` if the buffer is not frozen.
+    ///
+    /// # Notes
+    ///
+    /// This operation did nothing to the buffer. We use `&mut self` just for make
+    /// the API consistent with other APIs.
+    pub fn get(&mut self) -> Option<Bytes> {
+        self.frozen.clone()
+    }
+
+    // Advance the frozen buffer.
+    ///
+    /// # Panics
+    ///
+    /// Panic if the buffer is not frozen.
+    pub fn advance(&mut self, cnt: usize) {
+        debug_assert!(self.len == 0, "The buffer must be empty during advance");
+
+        let Some(bs) = self.frozen.as_mut() else {
+            unreachable!("It must be a bug to advance on not frozen buffer")
+        };
+        bs.advance(cnt);
+
+        if bs.is_empty() {
+            self.clean()
+        }
+    }
+
+    /// Cleanup the buffer, reset to the initial state.
+    #[inline]
+    pub fn clean(&mut self) {
+        self.frozen = None;
+        // This reserve cloud be cheap since we can reuse already allocated memory.
+        // (if all references to the frozen buffer are dropped)
+        self.buf.reserve(self.cap);
+    }
+}