opendal 0.1.6.pre.rc.1

data/core/src/raw/oio/entry.rs

@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::*;
+
+/// Entry is returned by `Page` or `BlockingPage` during list operations.
+///
+/// # Notes
+///
+/// Differences between `crate::Entry` and `oio::Entry`:
+///
+/// - `crate::Entry` is the user's public API and have less public methods.
+/// - `oio::Entry` is the raw API and doesn't expose to users.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Entry {
+    path: String,
+    meta: Metadata,
+}
+
+impl Entry {
+    /// Create a new entry by its corresponding underlying storage.
+    pub fn new(path: &str, meta: Metadata) -> Entry {
+        Self::with(path.to_string(), meta)
+    }
+
+    /// Create a new entry with given value.
+    pub fn with(mut path: String, meta: Metadata) -> Entry {
+        // Normalize path as `/` if it's empty.
+        if path.is_empty() {
+            path = "/".to_string();
+        }
+
+        debug_assert!(
+            meta.mode().is_dir() == path.ends_with('/'),
+            "mode {:?} not match with path {}",
+            meta.mode(),
+            path
+        );
+
+        Entry { path, meta }
+    }
+
+    /// Set path for entry.
+    pub fn set_path(&mut self, path: &str) -> &mut Self {
+        self.path = path.to_string();
+        self
+    }
+
+    /// Get the path of entry.
+    pub fn path(&self) -> &str {
+        &self.path
+    }
+
+    /// Set mode for entry.
+    ///
+    /// # Note
+    ///
+    /// Please use this function carefully.
+    pub fn set_mode(&mut self, mode: EntryMode) -> &mut Self {
+        self.meta.set_mode(mode);
+        self
+    }
+
+    /// Get entry's mode.
+    pub fn mode(&self) -> EntryMode {
+        self.meta.mode()
+    }
+
+    /// Consume self to convert into an Entry.
+    ///
+    /// NOTE: implement this by hand to avoid leaking raw entry to end-users.
+    pub(crate) fn into_entry(self) -> crate::Entry {
+        crate::Entry::new(self.path, self.meta)
+    }
+}

data/core/src/raw/oio/list/api.rs

@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::future::Future;
+use std::ops::DerefMut;
+
+use crate::raw::oio::Entry;
+use crate::raw::*;
+use crate::*;
+
+/// The boxed version of [`List`]
+pub type Lister = Box<dyn ListDyn>;
+
+/// Page trait is used by [`raw::Accessor`] to implement `list` operation.
+pub trait List: Unpin + Send + Sync {
+    /// Fetch a new page of [`Entry`]
+    ///
+    /// `Ok(None)` means all pages have been returned. Any following call
+    /// to `next` will always get the same result.
+    fn next(&mut self) -> impl Future<Output = Result<Option<Entry>>> + MaybeSend;
+}
+
+impl List for () {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        Ok(None)
+    }
+}
+
+impl<P: List> List for Option<P> {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        match self {
+            Some(p) => p.next().await,
+            None => Ok(None),
+        }
+    }
+}
+
+/// ListDyn is the dyn version of [`List`]. Makes it possible to use as
+/// `Box<dyn ListDyn>`.
+pub trait ListDyn: Unpin + Send + Sync {
+    /// The dyn version of [`List::next`].
+    fn next_dyn(&mut self) -> BoxedFuture<'_, Result<Option<Entry>>>;
+}
+
+impl<T: List + ?Sized> ListDyn for T {
+    fn next_dyn(&mut self) -> BoxedFuture<'_, Result<Option<Entry>>> {
+        Box::pin(self.next())
+    }
+}
+
+impl<T: ListDyn + ?Sized> List for Box<T> {
+    async fn next(&mut self) -> Result<Option<Entry>> {
+        self.deref_mut().next_dyn().await
+    }
+}

data/core/src/raw/oio/list/flat_list.rs

@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+
+/// FlatLister will walk dir in bottom up way:
+///
+/// - List nested dir first
+/// - Go back into parent dirs one by one
+///
+/// Given the following file tree:
+///
+/// ```txt
+/// .
+/// ├── dir_x/
+/// │   ├── dir_y/
+/// │   │   ├── dir_z/
+/// │   │   └── file_c
+/// │   └── file_b
+/// └── file_a
+/// ```
+///
+/// ToFlatLister will output entries like:
+///
+/// ```txt
+/// dir_x/dir_y/dir_z/file_c
+/// dir_x/dir_y/dir_z/
+/// dir_x/dir_y/file_b
+/// dir_x/dir_y/
+/// dir_x/file_a
+/// dir_x/
+/// ```
+///
+/// # Note
+///
+/// There is no guarantee about the order between files and dirs at the same level.
+/// We only make sure the nested dirs will show up before parent dirs.
+///
+/// Especially, for storage services that can't return dirs first, ToFlatLister
+/// may output parent dirs' files before nested dirs, this is expected because files
+/// always output directly while listing.
+pub struct FlatLister<A: Access, L> {
+    acc: A,
+
+    next_dir: Option<oio::Entry>,
+    active_lister: Vec<(Option<oio::Entry>, L)>,
+}
+
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this FlatLister.
+unsafe impl<A: Access, L> Send for FlatLister<A, L> {}
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FsLister.
+unsafe impl<A: Access, L> Sync for FlatLister<A, L> {}
+
+impl<A, L> FlatLister<A, L>
+where
+    A: Access,
+{
+    /// Create a new flat lister
+    pub fn new(acc: A, path: &str) -> FlatLister<A, L> {
+        FlatLister {
+            acc,
+            next_dir: Some(oio::Entry::new(path, Metadata::new(EntryMode::DIR))),
+            active_lister: vec![],
+        }
+    }
+}
+
+impl<A, L> oio::List for FlatLister<A, L>
+where
+    A: Access<Lister = L>,
+    L: oio::List,
+{
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        loop {
+            if let Some(de) = self.next_dir.take() {
+                let (_, mut l) = self.acc.list(de.path(), OpList::new()).await?;
+                if let Some(v) = l.next().await? {
+                    self.active_lister.push((Some(de.clone()), l));
+
+                    if v.mode().is_dir() {
+                        // should not loop itself again
+                        if v.path() != de.path() {
+                            self.next_dir = Some(v);
+                            continue;
+                        }
+                    } else {
+                        return Ok(Some(v));
+                    }
+                }
+            }
+
+            let (de, lister) = match self.active_lister.last_mut() {
+                Some((de, lister)) => (de, lister),
+                None => return Ok(None),
+            };
+
+            match lister.next().await? {
+                Some(v) if v.mode().is_dir() => {
+                    // should not loop itself again
+                    if v.path() != de.as_ref().expect("de should not be none here").path() {
+                        self.next_dir = Some(v);
+                        continue;
+                    }
+                }
+                Some(v) => return Ok(Some(v)),
+                None => match de.take() {
+                    Some(de) => {
+                        return Ok(Some(de));
+                    }
+                    None => {
+                        let _ = self.active_lister.pop();
+                        continue;
+                    }
+                },
+            }
+        }
+    }
+}

data/core/src/raw/oio/list/hierarchy_list.rs

@@ -0,0 +1,135 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashSet;
+
+use crate::raw::*;
+use crate::*;
+
+/// ToHierarchyLister will convert a flat list to hierarchy by filter
+/// not needed entries.
+///
+/// # Notes
+///
+/// ToHierarchyLister filter entries after fetch entries. So it's possible
+/// to return an empty vec. It doesn't mean the all pages have been
+/// returned.
+///
+/// Please keep calling next until we returned `Ok(None)`
+pub struct HierarchyLister<P> {
+    lister: P,
+    path: String,
+    visited: HashSet<String>,
+    recursive: bool,
+}
+
+impl<P> HierarchyLister<P> {
+    /// Create a new hierarchy lister
+    pub fn new(lister: P, path: &str, recursive: bool) -> HierarchyLister<P> {
+        let path = if path == "/" {
+            "".to_string()
+        } else {
+            path.to_string()
+        };
+
+        HierarchyLister {
+            lister,
+            path,
+            visited: HashSet::default(),
+            recursive,
+        }
+    }
+
+    /// ## NOTES
+    ///
+    /// We take `&mut Entry` here because we need to perform modification on entry in the case like
+    /// listing "a/" with existing key `a/b/c`.
+    ///
+    /// In this case, we got a key `a/b/c`, but we should return `a/b/` instead to keep the hierarchy.
+    fn keep_entry(&mut self, e: &mut oio::Entry) -> bool {
+        // If path is not started with prefix, drop it.
+        //
+        // Ideally, it should never happen. But we just tolerate
+        // this state.
+        if !e.path().starts_with(&self.path) {
+            return false;
+        }
+
+        // Don't return already visited path.
+        if self.visited.contains(e.path()) {
+            return false;
+        }
+
+        let prefix_len = self.path.len();
+
+        let idx = if let Some(idx) = e.path()[prefix_len..].find('/') {
+            idx + prefix_len + 1
+        } else {
+            // If there is no `/` in path, it's a normal file, we
+            // can return it directly.
+            return true;
+        };
+
+        // idx == path.len() means it's contain only one `/` at the
+        // end of path.
+        if idx == e.path().len() {
+            if !self.visited.contains(e.path()) {
+                self.visited.insert(e.path().to_string());
+            }
+            return true;
+        }
+
+        // If idx < path.len() means that are more levels to come.
+        // We should check the first dir path.
+        let has = {
+            let path = &e.path()[..idx];
+            self.visited.contains(path)
+        };
+        if !has {
+            let path = {
+                let path = &e.path()[..idx];
+                path.to_string()
+            };
+
+            e.set_path(&path);
+            e.set_mode(EntryMode::DIR);
+            self.visited.insert(path);
+
+            return true;
+        }
+
+        false
+    }
+}
+
+impl<P: oio::List> oio::List for HierarchyLister<P> {
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        loop {
+            let mut entry = match self.lister.next().await? {
+                Some(entry) => entry,
+                None => return Ok(None),
+            };
+
+            if self.recursive {
+                return Ok(Some(entry));
+            }
+            if self.keep_entry(&mut entry) {
+                return Ok(Some(entry));
+            }
+        }
+    }
+}

data/core/src/raw/oio/list/mod.rs

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod api;
+pub use api::List;
+pub use api::ListDyn;
+pub use api::Lister;
+
+mod page_list;
+pub use page_list::PageContext;
+pub use page_list::PageList;
+pub use page_list::PageLister;
+
+mod flat_list;
+pub use flat_list::FlatLister;
+
+mod hierarchy_list;
+pub use hierarchy_list::HierarchyLister;
+
+mod prefix_list;
+pub use prefix_list::PrefixLister;

data/core/src/raw/oio/list/page_list.rs

@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::VecDeque;
+use std::future::Future;
+
+use crate::raw::*;
+use crate::*;
+
+/// PageList is used to implement [`oio::List`] based on API supporting pagination. By implementing
+/// PageList, services don't need to care about the details of page list.
+///
+/// # Architecture
+///
+/// The architecture after adopting [`PageList`]:
+///
+/// - Services impl `PageList`
+/// - `PageLister` impl `List`
+/// - Expose `PageLister` as `Accessor::Lister`
+pub trait PageList: Send + Sync + Unpin + 'static {
+    /// next_page is used to fetch next page of entries from underlying storage.
+    #[cfg(not(target_arch = "wasm32"))]
+    fn next_page(&self, ctx: &mut PageContext) -> impl Future<Output = Result<()>> + MaybeSend;
+    #[cfg(target_arch = "wasm32")]
+    /// next_page is used to fetch next page of entries from underlying storage.
+    fn next_page(&self, ctx: &mut PageContext) -> impl Future<Output = Result<()>>;
+}
+
+/// PageContext is the context passing between `PageList`.
+///
+/// [`PageLister`] will init the PageContext, and implementer of [`PageList`] should fill the `PageContext`
+/// based on their needs.
+///
+/// - Set `done` to `true` if all page have been fetched.
+/// - Update `token` if there is more page to fetch. `token` is not exposed to users, it's internal used only.
+/// - Push back into the entries for each entry fetched from underlying storage.
+///
+/// NOTE: `entries` is a `VecDeque` to avoid unnecessary memory allocation. Only `push_back` is allowed.
+pub struct PageContext {
+    /// done is used to indicate whether the list operation is done.
+    pub done: bool,
+    /// token is used by underlying storage services to fetch next page.
+    pub token: String,
+    /// entries are used to store entries fetched from underlying storage.
+    ///
+    /// Please always reuse the same `VecDeque` to avoid unnecessary memory allocation.
+    /// PageLister makes sure that entries is reset before calling `next_page`. Implementer
+    /// can call `push_back` on `entries` directly.
+    pub entries: VecDeque<oio::Entry>,
+}
+
+/// PageLister implements [`oio::List`] based on [`PageList`].
+pub struct PageLister<L: PageList> {
+    inner: L,
+    ctx: PageContext,
+}
+
+impl<L> PageLister<L>
+where
+    L: PageList,
+{
+    /// Create a new PageLister.
+    pub fn new(l: L) -> Self {
+        Self {
+            inner: l,
+            ctx: PageContext {
+                done: false,
+                token: "".to_string(),
+                entries: VecDeque::new(),
+            },
+        }
+    }
+}
+
+impl<L> oio::List for PageLister<L>
+where
+    L: PageList,
+{
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        loop {
+            if let Some(entry) = self.ctx.entries.pop_front() {
+                return Ok(Some(entry));
+            }
+            if self.ctx.done {
+                return Ok(None);
+            }
+
+            self.inner.next_page(&mut self.ctx).await?;
+        }
+    }
+}

data/core/src/raw/oio/list/prefix_list.rs

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::raw::*;
+use crate::*;
+
+/// PrefixLister is used to filter entries by prefix.
+///
+/// For example, if we have a lister that returns entries:
+///
+/// ```txt
+/// .
+/// ├── file_a
+/// └── file_b
+/// ```
+///
+/// We can use `PrefixLister` to filter entries with prefix `file_`.
+pub struct PrefixLister<L> {
+    lister: L,
+    prefix: String,
+}
+
+/// # Safety
+///
+/// We will only take `&mut Self` reference for FsLister.
+unsafe impl<L> Sync for PrefixLister<L> {}
+
+impl<L> PrefixLister<L> {
+    /// Create a new flat lister
+    pub fn new(lister: L, prefix: &str) -> PrefixLister<L> {
+        PrefixLister {
+            lister,
+            prefix: prefix.to_string(),
+        }
+    }
+}
+
+impl<L> oio::List for PrefixLister<L>
+where
+    L: oio::List,
+{
+    async fn next(&mut self) -> Result<Option<oio::Entry>> {
+        loop {
+            match self.lister.next().await {
+                Ok(Some(e)) if !e.path().starts_with(&self.prefix) => continue,
+                v => return v,
+            }
+        }
+    }
+}

data/core/src/raw/oio/mod.rs

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! `oio` provides OpenDAL's raw traits and types that opendal returns as
+//! output.
+//!
+//! Those types should only be used internally and we don't want users to
+//! depend on them.
+
+mod delete;
+pub use delete::*;
+
+mod read;
+pub use read::*;
+
+mod write;
+pub use write::*;
+
+mod list;
+pub use list::*;
+
+mod entry;
+pub use entry::Entry;
+
+mod buf;
+pub use buf::*;