opendal 0.1.6.pre.rc.1

data/core/src/services/onedrive/lister.rs

@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+
+use super::core::OneDriveCore;
+use super::error::parse_error;
+use super::graph_model::GraphApiOneDriveListResponse;
+use super::graph_model::ItemType;
+use super::graph_model::GENERAL_SELECT_PARAM;
+use crate::raw::oio;
+use crate::raw::*;
+use crate::*;
+
+pub struct OneDriveLister {
+    core: Arc<OneDriveCore>,
+    path: String,
+    op: OpList,
+}
+
+impl OneDriveLister {
+    const DRIVE_ROOT_PREFIX: &'static str = "/drive/root:";
+
+    pub(crate) fn new(path: String, core: Arc<OneDriveCore>, args: &OpList) -> Self {
+        Self {
+            core,
+            path,
+            op: args.clone(),
+        }
+    }
+}
+
+impl oio::PageList for OneDriveLister {
+    async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
+        let request_url = if ctx.token.is_empty() {
+            let base = format!(
+                "{}:/children?{}",
+                self.core.onedrive_item_url(&self.path, true),
+                GENERAL_SELECT_PARAM
+            );
+            if let Some(limit) = self.op.limit() {
+                base + &format!("&$top={limit}")
+            } else {
+                base
+            }
+        } else {
+            ctx.token.clone()
+        };
+
+        let response = self.core.onedrive_get_next_list_page(&request_url).await?;
+
+        let status_code = response.status();
+        if !status_code.is_success() {
+            if status_code == http::StatusCode::NOT_FOUND {
+                ctx.done = true;
+                return Ok(());
+            }
+            return Err(parse_error(response));
+        }
+
+        let bytes = response.into_body();
+        let decoded_response: GraphApiOneDriveListResponse =
+            serde_json::from_reader(bytes.reader()).map_err(new_json_deserialize_error)?;
+
+        let list_with_versions = self.core.info.native_capability().list_with_versions;
+
+        // Include the current directory itself when handling the first page of the listing.
+        if ctx.token.is_empty() && !ctx.done {
+            // TODO: when listing a directory directly, we could reuse the stat result,
+            // cache the result when listing nested directory
+            let path = if self.path == "/" {
+                "".to_string()
+            } else {
+                self.path.clone()
+            };
+
+            let meta = self.core.onedrive_stat(&path, OpStat::default()).await?;
+
+            // skip `list_with_versions` intentionally because a folder doesn't have versions
+
+            let entry = oio::Entry::new(&path, meta);
+            ctx.entries.push_back(entry);
+        }
+
+        if let Some(next_link) = decoded_response.next_link {
+            ctx.token = next_link;
+        } else {
+            ctx.done = true;
+        }
+
+        for drive_item in decoded_response.value {
+            let name = drive_item.name;
+            let parent_path = drive_item.parent_reference.path;
+            let parent_path = parent_path
+                .strip_prefix(Self::DRIVE_ROOT_PREFIX)
+                .unwrap_or("");
+
+            let path = format!("{parent_path}/{name}");
+            let mut normalized_path = build_rel_path(self.core.root.as_str(), path.as_str());
+            let entry_mode = match drive_item.item_type {
+                ItemType::Folder { .. } => EntryMode::DIR,
+                ItemType::File { .. } => EntryMode::FILE,
+            };
+
+            // Add the trailing `/` because OneDrive returns a directory with the name
+            if entry_mode == EntryMode::DIR {
+                normalized_path.push('/');
+            }
+
+            let mut meta = Metadata::new(entry_mode)
+                .with_etag(drive_item.e_tag)
+                .with_content_length(drive_item.size.max(0) as u64);
+            let last_modified =
+                parse_datetime_from_rfc3339(drive_item.last_modified_date_time.as_str())?;
+            meta.set_last_modified(last_modified);
+
+            // When listing a directory with `$expand=versions`, OneDrive returns 400 "Operation not supported".
+            // Thus, `list_with_versions` induces N+1 requests. This N+1 is intentional.
+            // N+1 is horrendous but we can't do any better without OneDrive's API support.
+            // When OneDrive supports listing with versions API, remove this.
+            if list_with_versions {
+                let versions = self.core.onedrive_list_versions(&path).await?;
+                if let Some(version) = versions.first() {
+                    meta.set_version(&version.id);
+                }
+            }
+
+            let entry = oio::Entry::new(&normalized_path, meta);
+            ctx.entries.push_back(entry)
+        }
+
+        Ok(())
+    }
+}

data/core/src/services/onedrive/mod.rs

@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+/// Default scheme for onedrive service.
+#[cfg(feature = "services-onedrive")]
+pub(super) const DEFAULT_SCHEME: &str = "onedrive";
+#[cfg(feature = "services-onedrive")]
+mod backend;
+#[cfg(feature = "services-onedrive")]
+mod core;
+#[cfg(feature = "services-onedrive")]
+mod delete;
+#[cfg(feature = "services-onedrive")]
+mod error;
+#[cfg(feature = "services-onedrive")]
+mod graph_model;
+#[cfg(feature = "services-onedrive")]
+mod lister;
+#[cfg(feature = "services-onedrive")]
+mod writer;
+
+#[cfg(feature = "services-onedrive")]
+mod builder;
+#[cfg(feature = "services-onedrive")]
+pub use builder::OnedriveBuilder as Onedrive;
+
+mod config;
+pub use config::OnedriveConfig;

data/core/src/services/onedrive/writer.rs

@@ -0,0 +1,168 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use bytes::Buf;
+use bytes::Bytes;
+use http::StatusCode;
+
+use super::core::OneDriveCore;
+use super::error::parse_error;
+use super::graph_model::OneDriveItem;
+use super::graph_model::OneDriveUploadSessionCreationResponseBody;
+use crate::raw::*;
+use crate::*;
+
+pub struct OneDriveWriter {
+    core: Arc<OneDriveCore>,
+    op: OpWrite,
+    path: String,
+}
+
+impl OneDriveWriter {
+    const MAX_SIMPLE_SIZE: usize = 4 * 1024 * 1024; // 4MB
+
+    // OneDrive demands the chunk size to be a multiple to to 320 KiB.
+    // Choose a value smaller than `MAX_SIMPLE_SIZE`
+    const CHUNK_SIZE_FACTOR: usize = 327_680 * 12; // floor(MAX_SIMPLE_SIZE / 320KB)
+
+    pub fn new(core: Arc<OneDriveCore>, op: OpWrite, path: String) -> Self {
+        OneDriveWriter { core, op, path }
+    }
+}
+
+// OneDrive requires multipart writes to specify the total size of the file.
+// While OpenDAL supports multiple writes, due to OneDrive's limitation, we opt to
+// use `OneShotWrite` instead of `MultipartWrite`.
+impl oio::OneShotWrite for OneDriveWriter {
+    async fn write_once(&self, bs: Buffer) -> Result<Metadata> {
+        let size = bs.len();
+
+        let meta = if size <= Self::MAX_SIMPLE_SIZE {
+            self.write_simple(bs).await?
+        } else {
+            self.write_chunked(bs).await?
+        };
+
+        Ok(meta)
+    }
+}
+
+impl OneDriveWriter {
+    async fn write_simple(&self, bs: Buffer) -> Result<Metadata> {
+        let response = self
+            .core
+            .onedrive_upload_simple(&self.path, &self.op, bs)
+            .await?;
+
+        match response.status() {
+            StatusCode::CREATED | StatusCode::OK => {
+                let item: OneDriveItem = serde_json::from_reader(response.into_body().reader())
+                    .map_err(new_json_deserialize_error)?;
+
+                let mut meta = Metadata::new(EntryMode::FILE)
+                    .with_etag(item.e_tag)
+                    .with_content_length(item.size.max(0) as u64);
+
+                let last_modified = item.last_modified_date_time;
+                let date_utc_last_modified = parse_datetime_from_rfc3339(&last_modified)?;
+                meta.set_last_modified(date_utc_last_modified);
+
+                Ok(meta)
+            }
+            _ => Err(parse_error(response)),
+        }
+    }
+
+    pub(crate) async fn write_chunked(&self, bs: Buffer) -> Result<Metadata> {
+        // Upload large files via sessions: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession?view=odsp-graph-online#upload-bytes-to-the-upload-session
+        // 1. Create an upload session
+        // 2. Upload the bytes of each chunk
+        // 3. Commit the session
+
+        let session_response = self.create_upload_session().await?;
+
+        let mut offset = 0;
+        let total_bytes = bs.to_bytes();
+        let total_len = total_bytes.len();
+        let chunks = total_bytes.chunks(OneDriveWriter::CHUNK_SIZE_FACTOR);
+
+        for chunk in chunks {
+            let mut end = offset + OneDriveWriter::CHUNK_SIZE_FACTOR;
+            if end > total_bytes.len() {
+                end = total_bytes.len();
+            }
+            let chunk_end = end - 1;
+
+            let response = self
+                .core
+                .onedrive_chunked_upload(
+                    &session_response.upload_url,
+                    &self.op,
+                    offset,
+                    chunk_end,
+                    total_len,
+                    Buffer::from(Bytes::copy_from_slice(chunk)),
+                )
+                .await?;
+
+            match response.status() {
+                // Typical response code: 202 Accepted
+                // Reference: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online#response
+                StatusCode::ACCEPTED | StatusCode::OK => {} // skip, in the middle of upload
+                StatusCode::CREATED => {
+                    // last trunk
+                    let item: OneDriveItem = serde_json::from_reader(response.into_body().reader())
+                        .map_err(new_json_deserialize_error)?;
+
+                    let mut meta = Metadata::new(EntryMode::FILE)
+                        .with_etag(item.e_tag)
+                        .with_content_length(item.size.max(0) as u64);
+
+                    let last_modified = item.last_modified_date_time;
+                    let date_utc_last_modified = parse_datetime_from_rfc3339(&last_modified)?;
+                    meta.set_last_modified(date_utc_last_modified);
+                    return Ok(meta);
+                }
+                _ => return Err(parse_error(response)),
+            }
+
+            offset += OneDriveWriter::CHUNK_SIZE_FACTOR;
+        }
+
+        debug_assert!(false, "should have returned");
+
+        Ok(Metadata::default()) // should not happen, but start with handling this gracefully - do nothing, but return the default metadata
+    }
+
+    async fn create_upload_session(&self) -> Result<OneDriveUploadSessionCreationResponseBody> {
+        let response = self
+            .core
+            .onedrive_create_upload_session(&self.path, &self.op)
+            .await?;
+        match response.status() {
+            StatusCode::OK => {
+                let bs = response.into_body();
+                let result: OneDriveUploadSessionCreationResponseBody =
+                    serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
+                Ok(result)
+            }
+            _ => Err(parse_error(response)),
+        }
+    }
+}

data/core/src/services/opfs/backend.rs

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+use std::sync::Arc;
+use web_sys::FileSystemGetDirectoryOptions;
+
+use super::utils::*;
+use crate::raw::*;
+use crate::Result;
+
+/// OPFS Service backend
+#[derive(Default, Debug, Clone)]
+pub struct OpfsBackend {}
+
+impl Access for OpfsBackend {
+    type Reader = ();
+
+    type Writer = ();
+
+    type Lister = ();
+
+    type Deleter = ();
+
+    fn info(&self) -> Arc<AccessorInfo> {
+        Arc::new(AccessorInfo::default())
+    }
+
+    async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result<RpCreateDir> {
+        let opt = FileSystemGetDirectoryOptions::new();
+        opt.set_create(true);
+        get_directory_handle(path, &opt).await?;
+
+        Ok(RpCreateDir::default())
+    }
+}

data/core/src/services/opfs/config.rs

@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use serde::Deserialize;
+use serde::Serialize;
+
+/// Config for OPFS.
+#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[serde(default)]
+#[non_exhaustive]
+pub struct OPFSConfig {}

data/core/src/services/opfs/core.rs

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Debug;
+
+use wasm_bindgen::JsCast;
+use wasm_bindgen_futures::JsFuture;
+use web_sys::File;
+use web_sys::FileSystemWritableFileStream;
+
+use crate::Error;
+use crate::Result;
+
+use super::error::*;
+use super::utils::*;
+
+#[derive(Default, Debug)]
+pub struct OpfsCore {}
+
+impl OpfsCore {
+    #[allow(unused)]
+    async fn store_file(&self, file_name: &str, content: &[u8]) -> Result<(), Error> {
+        let handle = get_handle_by_filename(file_name).await?;
+
+        let writable: FileSystemWritableFileStream = JsFuture::from(handle.create_writable())
+            .await
+            .and_then(JsCast::dyn_into)
+            .map_err(parse_js_error)?;
+
+        // QuotaExceeded or NotAllowed
+        JsFuture::from(
+            writable
+                .write_with_u8_array(content)
+                .map_err(parse_js_error)?,
+        )
+        .await
+        .map_err(parse_js_error)?;
+
+        JsFuture::from(writable.close())
+            .await
+            .map_err(parse_js_error)?;
+
+        Ok(())
+    }
+
+    #[allow(unused)]
+    async fn read_file(&self, file_name: &str) -> Result<Vec<u8>, Error> {
+        let handle = get_handle_by_filename(file_name).await?;
+
+        let file: File = JsFuture::from(handle.get_file())
+            .await
+            .and_then(JsCast::dyn_into)
+            .map_err(parse_js_error)?;
+        let array_buffer = JsFuture::from(file.array_buffer())
+            .await
+            .map_err(parse_js_error)?;
+
+        Ok(js_sys::Uint8Array::new(&array_buffer).to_vec())
+    }
+}

data/core/src/services/opfs/docs.md

@@ -0,0 +1,18 @@
+## Capabilities
+
+This service can be used to:
+
+- [ ] stat
+- [ ] read
+- [ ] write
+- [ ] create_dir
+- [ ] delete
+- [ ] copy
+- [ ] rename
+- [ ] list
+- [ ] presign
+- [ ] blocking
+
+## Configuration
+
+## Example

data/core/src/services/opfs/error.rs

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use wasm_bindgen::JsValue;
+
+use crate::{Error, ErrorKind};
+
+pub(crate) fn parse_js_error(msg: JsValue) -> Error {
+    Error::new(
+        ErrorKind::Unexpected,
+        msg.as_string().unwrap_or_else(String::new),
+    )
+}

data/core/src/services/opfs/mod.rs

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "services-opfs")]
+mod backend;
+#[cfg(feature = "services-opfs")]
+mod core;
+
+#[cfg(feature = "services-opfs")]
+mod config;
+
+#[cfg(feature = "services-opfs")]
+mod error;
+
+#[cfg(feature = "services-opfs")]
+mod utils;

data/core/src/services/opfs/utils.rs

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::Result;
+use wasm_bindgen::JsCast;
+use wasm_bindgen_futures::JsFuture;
+use web_sys::{
+    window, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions,
+    FileSystemGetFileOptions,
+};
+
+use super::error::*;
+
+pub(crate) async fn get_root_directory_handle() -> Result<FileSystemDirectoryHandle> {
+    let navigator = window().unwrap().navigator();
+    let storage_manager = navigator.storage();
+    JsFuture::from(storage_manager.get_directory())
+        .await
+        .and_then(JsCast::dyn_into)
+        .map_err(parse_js_error)
+}
+
+pub(crate) async fn get_directory_handle(
+    dir: &str,
+    dir_opt: &FileSystemGetDirectoryOptions,
+) -> Result<FileSystemDirectoryHandle> {
+    let dirs: Vec<&str> = dir.trim_matches('/').split('/').collect();
+
+    let mut handle = get_root_directory_handle().await?;
+    for dir in dirs {
+        handle = JsFuture::from(handle.get_directory_handle_with_options(dir, dir_opt))
+            .await
+            .and_then(JsCast::dyn_into)
+            .map_err(parse_js_error)?;
+    }
+
+    Ok(handle)
+}
+
+pub(crate) async fn get_handle_by_filename(filename: &str) -> Result<FileSystemFileHandle> {
+    let navigator = window().unwrap().navigator();
+    let storage_manager = navigator.storage();
+    let root: FileSystemDirectoryHandle = JsFuture::from(storage_manager.get_directory())
+        .await
+        .and_then(JsCast::dyn_into)
+        .map_err(parse_js_error)?;
+
+    // maybe the option should be exposed?
+    let opt = FileSystemGetFileOptions::new();
+    opt.set_create(true);
+
+    JsFuture::from(root.get_file_handle_with_options(filename, &opt))
+        .await
+        .and_then(JsCast::dyn_into)
+        .map_err(parse_js_error)
+}