opendal 0.1.6.pre.rc.1

data/core/src/raw/http_util/body.rs

@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::cmp::Ordering;
+
+use futures::Stream;
+use futures::StreamExt;
+use oio::Read;
+
+use crate::raw::*;
+use crate::*;
+
+/// The streaming body that OpenDAL's HttpClient returned.
+///
+/// We implement [`oio::Read`] for the `HttpBody`. Services can use `HttpBody` as
+/// [`Access::Read`].
+pub struct HttpBody {
+    #[cfg(not(target_arch = "wasm32"))]
+    stream: Box<dyn Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static>,
+    #[cfg(target_arch = "wasm32")]
+    stream: Box<dyn Stream<Item = Result<Buffer>> + Unpin + 'static>,
+    size: Option<u64>,
+    consumed: u64,
+}
+
+/// # Safety
+///
+/// HttpBody is `Send` on non wasm32 targets.
+unsafe impl Send for HttpBody {}
+
+/// # Safety
+///
+/// HttpBody is sync on non wasm32 targets.
+unsafe impl Sync for HttpBody {}
+
+impl HttpBody {
+    /// Create a new `HttpBody` with given stream and optional size.
+    #[cfg(not(target_arch = "wasm32"))]
+    pub fn new<S>(stream: S, size: Option<u64>) -> Self
+    where
+        S: Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static,
+    {
+        HttpBody {
+            stream: Box::new(stream),
+            size,
+            consumed: 0,
+        }
+    }
+
+    /// Create a new `HttpBody` with given stream and optional size.
+    #[cfg(target_arch = "wasm32")]
+    pub fn new<S>(stream: S, size: Option<u64>) -> Self
+    where
+        S: Stream<Item = Result<Buffer>> + Unpin + 'static,
+    {
+        HttpBody {
+            stream: Box::new(stream),
+            size,
+            consumed: 0,
+        }
+    }
+
+    /// Map the inner stream.
+    #[cfg(not(target_arch = "wasm32"))]
+    pub(crate) fn map_inner(
+        mut self,
+        f: impl FnOnce(
+            Box<dyn Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static>,
+        )
+            -> Box<dyn Stream<Item = Result<Buffer>> + Send + Sync + Unpin + 'static>,
+    ) -> Self {
+        self.stream = f(self.stream);
+        self
+    }
+
+    /// Map the inner stream.
+    #[cfg(target_arch = "wasm32")]
+    pub(crate) fn map_inner(
+        mut self,
+        f: impl FnOnce(
+            Box<dyn Stream<Item = Result<Buffer>> + Unpin + 'static>,
+        ) -> Box<dyn Stream<Item = Result<Buffer>> + Unpin + 'static>,
+    ) -> Self {
+        self.stream = f(self.stream);
+        self
+    }
+
+    /// Check if the consumed data is equal to the expected content length.
+    #[inline]
+    fn check(&self) -> Result<()> {
+        let Some(expect) = self.size else {
+            return Ok(());
+        };
+
+        let actual = self.consumed;
+        match actual.cmp(&expect) {
+            Ordering::Equal => Ok(()),
+            Ordering::Less => Err(Error::new(
+                ErrorKind::Unexpected,
+                format!("http response got too little data, expect: {expect}, actual: {actual}"),
+            )
+            .set_temporary()),
+            Ordering::Greater => Err(Error::new(
+                ErrorKind::Unexpected,
+                format!("http response got too much data, expect: {expect}, actual: {actual}"),
+            )
+            .set_temporary()),
+        }
+    }
+
+    /// Read all data from the stream.
+    pub async fn to_buffer(&mut self) -> Result<Buffer> {
+        self.read_all().await
+    }
+}
+
+impl oio::Read for HttpBody {
+    async fn read(&mut self) -> Result<Buffer> {
+        match self.stream.next().await.transpose()? {
+            Some(buf) => {
+                self.consumed += buf.len() as u64;
+                Ok(buf)
+            }
+            None => {
+                self.check()?;
+                Ok(Buffer::new())
+            }
+        }
+    }
+}

data/core/src/raw/http_util/bytes_content_range.rs

@@ -0,0 +1,239 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Display;
+use std::fmt::Formatter;
+use std::ops::Range;
+use std::ops::RangeInclusive;
+use std::str::FromStr;
+
+use crate::*;
+
+/// BytesContentRange is the content range of bytes.
+///
+/// `<unit>` should always be `bytes`.
+///
+/// ```text
+/// Content-Range: bytes <range-start>-<range-end>/<size>
+/// Content-Range: bytes <range-start>-<range-end>/*
+/// Content-Range: bytes */<size>
+/// ```
+///
+/// # Notes
+///
+/// ## Usage of the default.
+///
+/// `BytesContentRange::default` is not a valid content range.
+/// Please make sure their comes up with `with_range` or `with_size` call.
+///
+/// ## Allow clippy::len_without_is_empty
+///
+/// BytesContentRange implements `len()` but not `is_empty()` because it's useless.
+/// - When BytesContentRange's range is known, it must be non-empty.
+/// - When BytesContentRange's range is no known, we don't know whether it's empty.
+#[allow(clippy::len_without_is_empty)]
+#[derive(Debug, Default, Clone, Copy, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct BytesContentRange(
+    /// Start position of the range. `None` means unknown.
+    Option<u64>,
+    /// End position of the range. `None` means unknown.
+    Option<u64>,
+    /// Size of the whole content. `None` means unknown.
+    Option<u64>,
+);
+
+impl BytesContentRange {
+    /// Update BytesContentRange with range.
+    ///
+    /// The range is inclusive: `[start..=end]` as described in `content-range`.
+    pub fn with_range(mut self, start: u64, end: u64) -> Self {
+        self.0 = Some(start);
+        self.1 = Some(end);
+        self
+    }
+
+    /// Update BytesContentRange with size.
+    pub fn with_size(mut self, size: u64) -> Self {
+        self.2 = Some(size);
+        self
+    }
+
+    /// Get the length that specified by this BytesContentRange, return `None` if range is not known.
+    pub fn len(&self) -> Option<u64> {
+        if let (Some(start), Some(end)) = (self.0, self.1) {
+            Some(end - start + 1)
+        } else {
+            None
+        }
+    }
+
+    /// Get the size of this BytesContentRange, return `None` if size is not known.
+    pub fn size(&self) -> Option<u64> {
+        self.2
+    }
+
+    /// Get the range inclusive of this BytesContentRange, return `None` if range is not known.
+    pub fn range(&self) -> Option<Range<u64>> {
+        if let (Some(start), Some(end)) = (self.0, self.1) {
+            Some(start..end + 1)
+        } else {
+            None
+        }
+    }
+
+    /// Get the range inclusive of this BytesContentRange, return `None` if range is not known.
+    pub fn range_inclusive(&self) -> Option<RangeInclusive<u64>> {
+        if let (Some(start), Some(end)) = (self.0, self.1) {
+            Some(start..=end)
+        } else {
+            None
+        }
+    }
+
+    /// Convert bytes content range into Content-Range header.
+    pub fn to_header(&self) -> String {
+        format!("bytes {self}")
+    }
+}
+
+impl Display for BytesContentRange {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match (self.0, self.1, self.2) {
+            (Some(start), Some(end), Some(size)) => write!(f, "{start}-{end}/{size}"),
+            (Some(start), Some(end), None) => write!(f, "{start}-{end}/*"),
+            (None, None, Some(size)) => write!(f, "*/{size}"),
+            _ => unreachable!("invalid bytes range: {:?}", self),
+        }
+    }
+}
+
+impl FromStr for BytesContentRange {
+    type Err = Error;
+
+    fn from_str(value: &str) -> Result<Self> {
+        let s = value.strip_prefix("bytes ").ok_or_else(|| {
+            Error::new(ErrorKind::Unexpected, "header content range is invalid")
+                .with_operation("BytesContentRange::from_str")
+                .with_context("value", value)
+        })?;
+
+        let parse_int_error = |e: std::num::ParseIntError| {
+            Error::new(ErrorKind::Unexpected, "header content range is invalid")
+                .with_operation("BytesContentRange::from_str")
+                .with_context("value", value)
+                .set_source(e)
+        };
+
+        if let Some(size) = s.strip_prefix("*/") {
+            return Ok(
+                BytesContentRange::default().with_size(size.parse().map_err(parse_int_error)?)
+            );
+        }
+
+        let s: Vec<_> = s.split('/').collect();
+        if s.len() != 2 {
+            return Err(
+                Error::new(ErrorKind::Unexpected, "header content range is invalid")
+                    .with_operation("BytesContentRange::from_str")
+                    .with_context("value", value),
+            );
+        }
+
+        let v: Vec<_> = s[0].split('-').collect();
+        if v.len() != 2 {
+            return Err(
+                Error::new(ErrorKind::Unexpected, "header content range is invalid")
+                    .with_operation("BytesContentRange::from_str")
+                    .with_context("value", value),
+            );
+        }
+        let start: u64 = v[0].parse().map_err(parse_int_error)?;
+        let end: u64 = v[1].parse().map_err(parse_int_error)?;
+        let mut bcr = BytesContentRange::default().with_range(start, end);
+
+        // Handle size part first.
+        if s[1] != "*" {
+            bcr = bcr.with_size(s[1].parse().map_err(parse_int_error)?)
+        };
+
+        Ok(bcr)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_bytes_content_range_from_str() -> Result<()> {
+        let cases = vec![
+            (
+                "range start with unknown size",
+                "bytes 123-123/*",
+                BytesContentRange::default().with_range(123, 123),
+            ),
+            (
+                "range start with known size",
+                "bytes 123-123/1024",
+                BytesContentRange::default()
+                    .with_range(123, 123)
+                    .with_size(1024),
+            ),
+            (
+                "only have size",
+                "bytes */1024",
+                BytesContentRange::default().with_size(1024),
+            ),
+        ];
+
+        for (name, input, expected) in cases {
+            let actual = input.parse()?;
+
+            assert_eq!(expected, actual, "{name}")
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_bytes_content_range_to_string() {
+        let h = BytesContentRange::default().with_size(1024);
+        assert_eq!(h.to_string(), "*/1024");
+
+        let h = BytesContentRange::default().with_range(0, 1023);
+        assert_eq!(h.to_string(), "0-1023/*");
+
+        let h = BytesContentRange::default()
+            .with_range(0, 1023)
+            .with_size(1024);
+        assert_eq!(h.to_string(), "0-1023/1024");
+    }
+
+    #[test]
+    fn test_bytes_content_range_to_header() {
+        let h = BytesContentRange::default().with_size(1024);
+        assert_eq!(h.to_header(), "bytes */1024");
+
+        let h = BytesContentRange::default().with_range(0, 1023);
+        assert_eq!(h.to_header(), "bytes 0-1023/*");
+
+        let h = BytesContentRange::default()
+            .with_range(0, 1023)
+            .with_size(1024);
+        assert_eq!(h.to_header(), "bytes 0-1023/1024");
+    }
+}

data/core/src/raw/http_util/bytes_range.rs

@@ -0,0 +1,260 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::fmt::Display;
+use std::fmt::Formatter;
+use std::ops::Bound;
+use std::ops::RangeBounds;
+use std::str::FromStr;
+
+use crate::*;
+
+/// BytesRange(offset, size) carries a range of content.
+///
+/// BytesRange implements `ToString` which can be used as `Range` HTTP header directly.
+///
+/// `<unit>` should always be `bytes`.
+///
+/// ```text
+/// Range: bytes=<range-start>-
+/// Range: bytes=<range-start>-<range-end>
+/// ```
+///
+/// # Notes
+///
+/// We don't support tailing read like `Range: bytes=-<range-end>`
+#[derive(Default, Debug, Clone, Copy, Eq, PartialEq)]
+pub struct BytesRange(
+    /// Offset of the range.
+    u64,
+    /// Size of the range.
+    Option<u64>,
+);
+
+impl BytesRange {
+    /// Create a new `BytesRange`
+    ///
+    /// It better to use `BytesRange::from(1024..2048)` to construct.
+    ///
+    /// # Note
+    ///
+    /// The behavior for `None` and `Some` of `size` is different.
+    ///
+    /// - size=None => `bytes=<offset>-`, read from `<offset>` until the end
+    /// - size=Some(1024) => `bytes=<offset>-<offset + 1024>`, read 1024 bytes starting from the `<offset>`
+    pub fn new(offset: u64, size: Option<u64>) -> Self {
+        BytesRange(offset, size)
+    }
+
+    /// Get offset of BytesRange.
+    pub fn offset(&self) -> u64 {
+        self.0
+    }
+
+    /// Get size of BytesRange.
+    pub fn size(&self) -> Option<u64> {
+        self.1
+    }
+
+    /// Advance the range by `n` bytes.
+    ///
+    /// # Panics
+    ///
+    /// Panic if input `n` is larger than the size of the range.
+    pub fn advance(&mut self, n: u64) {
+        self.0 += n;
+        self.1 = self.1.map(|size| size - n);
+    }
+
+    /// Check if this range is full of this content.
+    ///
+    /// If this range is full, we don't need to specify it in http request.
+    pub fn is_full(&self) -> bool {
+        self.0 == 0 && self.1.is_none()
+    }
+
+    /// Convert bytes range into Range header.
+    pub fn to_header(&self) -> String {
+        format!("bytes={self}")
+    }
+
+    /// Convert bytes range into rust range.
+    pub fn to_range(&self) -> impl RangeBounds<u64> {
+        (
+            Bound::Included(self.0),
+            match self.1 {
+                Some(size) => Bound::Excluded(self.0 + size),
+                None => Bound::Unbounded,
+            },
+        )
+    }
+
+    /// Convert bytes range into rust range with usize.
+    pub(crate) fn to_range_as_usize(self) -> impl RangeBounds<usize> {
+        (
+            Bound::Included(self.0 as usize),
+            match self.1 {
+                Some(size) => Bound::Excluded((self.0 + size) as usize),
+                None => Bound::Unbounded,
+            },
+        )
+    }
+}
+
+impl Display for BytesRange {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self.1 {
+            None => write!(f, "{}-", self.0),
+            Some(size) => write!(f, "{}-{}", self.0, self.0 + size - 1),
+        }
+    }
+}
+
+impl FromStr for BytesRange {
+    type Err = Error;
+
+    fn from_str(value: &str) -> Result<Self> {
+        let s = value.strip_prefix("bytes=").ok_or_else(|| {
+            Error::new(ErrorKind::Unexpected, "header range is invalid")
+                .with_operation("BytesRange::from_str")
+                .with_context("value", value)
+        })?;
+
+        if s.contains(',') {
+            return Err(Error::new(ErrorKind::Unexpected, "header range is invalid")
+                .with_operation("BytesRange::from_str")
+                .with_context("value", value));
+        }
+
+        let v = s.split('-').collect::<Vec<_>>();
+        if v.len() != 2 {
+            return Err(Error::new(ErrorKind::Unexpected, "header range is invalid")
+                .with_operation("BytesRange::from_str")
+                .with_context("value", value));
+        }
+
+        let parse_int_error = |e: std::num::ParseIntError| {
+            Error::new(ErrorKind::Unexpected, "header range is invalid")
+                .with_operation("BytesRange::from_str")
+                .with_context("value", value)
+                .set_source(e)
+        };
+
+        if v[1].is_empty() {
+            // <range-start>-
+            Ok(BytesRange::new(
+                v[0].parse().map_err(parse_int_error)?,
+                None,
+            ))
+        } else if v[0].is_empty() {
+            // -<suffix-length>
+            Err(Error::new(
+                ErrorKind::Unexpected,
+                "header range with tailing is not supported",
+            )
+            .with_operation("BytesRange::from_str")
+            .with_context("value", value))
+        } else {
+            // <range-start>-<range-end>
+            let start: u64 = v[0].parse().map_err(parse_int_error)?;
+            let end: u64 = v[1].parse().map_err(parse_int_error)?;
+            Ok(BytesRange::new(start, Some(end - start + 1)))
+        }
+    }
+}
+
+impl<T> From<T> for BytesRange
+where
+    T: RangeBounds<u64>,
+{
+    fn from(range: T) -> Self {
+        let offset = match range.start_bound().cloned() {
+            Bound::Included(n) => n,
+            Bound::Excluded(n) => n + 1,
+            Bound::Unbounded => 0,
+        };
+        let size = match range.end_bound().cloned() {
+            Bound::Included(n) => Some(n + 1 - offset),
+            Bound::Excluded(n) => Some(n - offset),
+            Bound::Unbounded => None,
+        };
+
+        BytesRange(offset, size)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_bytes_range_to_string() {
+        let h = BytesRange::new(0, Some(1024));
+        assert_eq!(h.to_string(), "0-1023");
+
+        let h = BytesRange::new(1024, None);
+        assert_eq!(h.to_string(), "1024-");
+
+        let h = BytesRange::new(1024, Some(1024));
+        assert_eq!(h.to_string(), "1024-2047");
+    }
+
+    #[test]
+    fn test_bytes_range_to_header() {
+        let h = BytesRange::new(0, Some(1024));
+        assert_eq!(h.to_header(), "bytes=0-1023");
+
+        let h = BytesRange::new(1024, None);
+        assert_eq!(h.to_header(), "bytes=1024-");
+
+        let h = BytesRange::new(1024, Some(1024));
+        assert_eq!(h.to_header(), "bytes=1024-2047");
+    }
+
+    #[test]
+    fn test_bytes_range_from_range_bounds() {
+        assert_eq!(BytesRange::new(0, None), BytesRange::from(..));
+        assert_eq!(BytesRange::new(10, None), BytesRange::from(10..));
+        assert_eq!(BytesRange::new(0, Some(11)), BytesRange::from(..=10));
+        assert_eq!(BytesRange::new(0, Some(10)), BytesRange::from(..10));
+        assert_eq!(BytesRange::new(10, Some(10)), BytesRange::from(10..20));
+        assert_eq!(BytesRange::new(10, Some(11)), BytesRange::from(10..=20));
+    }
+
+    #[test]
+    fn test_bytes_range_from_str() -> Result<()> {
+        let cases = vec![
+            ("range-start", "bytes=123-", BytesRange::new(123, None)),
+            ("range", "bytes=123-124", BytesRange::new(123, Some(2))),
+            ("one byte", "bytes=0-0", BytesRange::new(0, Some(1))),
+            (
+                "lower case header",
+                "bytes=0-0",
+                BytesRange::new(0, Some(1)),
+            ),
+        ];
+
+        for (name, input, expected) in cases {
+            let actual = input.parse()?;
+
+            assert_eq!(expected, actual, "{name}")
+        }
+
+        Ok(())
+    }
+}