opendal 0.1.6.pre.rc.1-arm64-darwin-23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +20 -0
- data/.tool-versions +1 -0
- data/.yardopts +1 -0
- data/DEPENDENCIES.md +9 -0
- data/DEPENDENCIES.rust.tsv +277 -0
- data/Gemfile +35 -0
- data/README.md +159 -0
- data/Rakefile +149 -0
- data/core/CHANGELOG.md +4929 -0
- data/core/CONTRIBUTING.md +61 -0
- data/core/DEPENDENCIES.md +3 -0
- data/core/DEPENDENCIES.rust.tsv +185 -0
- data/core/LICENSE +201 -0
- data/core/README.md +228 -0
- data/core/benches/README.md +18 -0
- data/core/benches/ops/README.md +26 -0
- data/core/benches/types/README.md +9 -0
- data/core/benches/vs_fs/README.md +35 -0
- data/core/benches/vs_s3/README.md +55 -0
- data/core/edge/README.md +3 -0
- data/core/edge/file_write_on_full_disk/README.md +14 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
- data/core/edge/s3_read_on_wasm/.gitignore +3 -0
- data/core/edge/s3_read_on_wasm/README.md +42 -0
- data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
- data/core/examples/README.md +23 -0
- data/core/examples/basic/README.md +15 -0
- data/core/examples/concurrent-upload/README.md +15 -0
- data/core/examples/multipart-upload/README.md +15 -0
- data/core/fuzz/.gitignore +5 -0
- data/core/fuzz/README.md +68 -0
- data/core/src/docs/comparisons/vs_object_store.md +183 -0
- data/core/src/docs/performance/concurrent_write.md +101 -0
- data/core/src/docs/performance/http_optimization.md +124 -0
- data/core/src/docs/rfcs/0000_example.md +74 -0
- data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
- data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
- data/core/src/docs/rfcs/0044_error_handle.md +198 -0
- data/core/src/docs/rfcs/0057_auto_region.md +160 -0
- data/core/src/docs/rfcs/0069_object_stream.md +145 -0
- data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
- data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
- data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
- data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
- data/core/src/docs/rfcs/0221_create_dir.md +89 -0
- data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
- data/core/src/docs/rfcs/0293_object_id.md +67 -0
- data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
- data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
- data/core/src/docs/rfcs/0413_presign.md +154 -0
- data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
- data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
- data/core/src/docs/rfcs/0438_multipart.md +163 -0
- data/core/src/docs/rfcs/0443_gateway.md +73 -0
- data/core/src/docs/rfcs/0501_new_builder.md +111 -0
- data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
- data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
- data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
- data/core/src/docs/rfcs/0623_redis_service.md +300 -0
- data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
- data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
- data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
- data/core/src/docs/rfcs/0926_object_reader.md +93 -0
- data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
- data/core/src/docs/rfcs/1085_object_handler.md +73 -0
- data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
- data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
- data/core/src/docs/rfcs/1420_object_writer.md +147 -0
- data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
- data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
- data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
- data/core/src/docs/rfcs/2133_append_api.md +88 -0
- data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
- data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
- data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
- data/core/src/docs/rfcs/2774_lister_api.md +66 -0
- data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
- data/core/src/docs/rfcs/2852_native_capability.md +58 -0
- data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
- data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
- data/core/src/docs/rfcs/3197_config.md +237 -0
- data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
- data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
- data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
- data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
- data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
- data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
- data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
- data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
- data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
- data/core/src/docs/rfcs/4638_executor.md +215 -0
- data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
- data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
- data/core/src/docs/rfcs/5479_context.md +140 -0
- data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
- data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
- data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
- data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
- data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
- data/core/src/docs/rfcs/6209_glob_support.md +132 -0
- data/core/src/docs/rfcs/6213_options_api.md +142 -0
- data/core/src/docs/rfcs/README.md +62 -0
- data/core/src/docs/upgrade.md +1556 -0
- data/core/src/services/aliyun_drive/docs.md +61 -0
- data/core/src/services/alluxio/docs.md +45 -0
- data/core/src/services/azblob/docs.md +77 -0
- data/core/src/services/azdls/docs.md +73 -0
- data/core/src/services/azfile/docs.md +65 -0
- data/core/src/services/b2/docs.md +54 -0
- data/core/src/services/cacache/docs.md +38 -0
- data/core/src/services/cloudflare_kv/docs.md +21 -0
- data/core/src/services/cos/docs.md +55 -0
- data/core/src/services/d1/docs.md +48 -0
- data/core/src/services/dashmap/docs.md +38 -0
- data/core/src/services/dbfs/docs.md +57 -0
- data/core/src/services/dropbox/docs.md +64 -0
- data/core/src/services/etcd/docs.md +45 -0
- data/core/src/services/foundationdb/docs.md +42 -0
- data/core/src/services/fs/docs.md +49 -0
- data/core/src/services/ftp/docs.md +42 -0
- data/core/src/services/gcs/docs.md +76 -0
- data/core/src/services/gdrive/docs.md +65 -0
- data/core/src/services/ghac/docs.md +84 -0
- data/core/src/services/github/docs.md +52 -0
- data/core/src/services/gridfs/docs.md +46 -0
- data/core/src/services/hdfs/docs.md +140 -0
- data/core/src/services/hdfs_native/docs.md +35 -0
- data/core/src/services/http/docs.md +45 -0
- data/core/src/services/huggingface/docs.md +61 -0
- data/core/src/services/ipfs/docs.md +45 -0
- data/core/src/services/ipmfs/docs.md +14 -0
- data/core/src/services/koofr/docs.md +51 -0
- data/core/src/services/lakefs/docs.md +62 -0
- data/core/src/services/memcached/docs.md +47 -0
- data/core/src/services/memory/docs.md +36 -0
- data/core/src/services/mini_moka/docs.md +19 -0
- data/core/src/services/moka/docs.md +42 -0
- data/core/src/services/mongodb/docs.md +49 -0
- data/core/src/services/monoiofs/docs.md +46 -0
- data/core/src/services/mysql/docs.md +47 -0
- data/core/src/services/obs/docs.md +54 -0
- data/core/src/services/onedrive/docs.md +115 -0
- data/core/src/services/opfs/docs.md +18 -0
- data/core/src/services/oss/docs.md +74 -0
- data/core/src/services/pcloud/docs.md +51 -0
- data/core/src/services/persy/docs.md +43 -0
- data/core/src/services/postgresql/docs.md +47 -0
- data/core/src/services/redb/docs.md +41 -0
- data/core/src/services/redis/docs.md +43 -0
- data/core/src/services/rocksdb/docs.md +54 -0
- data/core/src/services/s3/compatible_services.md +126 -0
- data/core/src/services/s3/docs.md +244 -0
- data/core/src/services/seafile/docs.md +54 -0
- data/core/src/services/sftp/docs.md +49 -0
- data/core/src/services/sled/docs.md +39 -0
- data/core/src/services/sqlite/docs.md +46 -0
- data/core/src/services/surrealdb/docs.md +54 -0
- data/core/src/services/swift/compatible_services.md +53 -0
- data/core/src/services/swift/docs.md +52 -0
- data/core/src/services/tikv/docs.md +43 -0
- data/core/src/services/upyun/docs.md +51 -0
- data/core/src/services/vercel_artifacts/docs.md +40 -0
- data/core/src/services/vercel_blob/docs.md +45 -0
- data/core/src/services/webdav/docs.md +49 -0
- data/core/src/services/webhdfs/docs.md +90 -0
- data/core/src/services/yandex_disk/docs.md +45 -0
- data/core/tests/behavior/README.md +77 -0
- data/core/tests/data/normal_dir/.gitkeep +0 -0
- data/core/tests/data/normal_file.txt +1041 -0
- data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
- data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
- data/core/users.md +13 -0
- data/extconf.rb +24 -0
- data/lib/opendal.rb +25 -0
- data/lib/opendal_ruby/entry.rb +35 -0
- data/lib/opendal_ruby/io.rb +70 -0
- data/lib/opendal_ruby/metadata.rb +44 -0
- data/lib/opendal_ruby/opendal_ruby.bundle +0 -0
- data/lib/opendal_ruby/operator.rb +29 -0
- data/lib/opendal_ruby/operator_info.rb +26 -0
- data/opendal.gemspec +91 -0
- data/test/blocking_op_test.rb +112 -0
- data/test/capability_test.rb +42 -0
- data/test/io_test.rb +172 -0
- data/test/lister_test.rb +77 -0
- data/test/metadata_test.rb +78 -0
- data/test/middlewares_test.rb +46 -0
- data/test/operator_info_test.rb +35 -0
- data/test/test_helper.rb +36 -0
- metadata +240 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
- Proposal Name: `limited_reader`
|
|
2
|
+
- Start Date: 2022-03-02
|
|
3
|
+
- RFC PR: [apache/opendal#0090](https://github.com/apache/opendal/pull/0090)
|
|
4
|
+
- Tracking Issue: [apache/opendal#0090](https://github.com/apache/opendal/issues/0090)
|
|
5
|
+
|
|
6
|
+
# Summary
|
|
7
|
+
|
|
8
|
+
Native support for the limited reader.
|
|
9
|
+
|
|
10
|
+
# Motivation
|
|
11
|
+
|
|
12
|
+
In proposal [object-native-api](./0041-object-native-api.md) we introduced `Reader`, in which we will send request like:
|
|
13
|
+
|
|
14
|
+
```rust
|
|
15
|
+
let op = OpRead {
|
|
16
|
+
path: self.path.to_string(),
|
|
17
|
+
offset: Some(self.current_offset()),
|
|
18
|
+
size: None,
|
|
19
|
+
};
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
In this implementation, we depend on the HTTP client to drop the request when we stop reading. However, we always read too much extra data, which decreases our reading performance.
|
|
23
|
+
|
|
24
|
+
Here is a benchmark around reading the whole file and only reading half:
|
|
25
|
+
|
|
26
|
+
```txt
|
|
27
|
+
s3/read/1c741003-40ef-43a9-b23f-b6a32ed7c4c6
|
|
28
|
+
time: [7.2697 ms 7.3521 ms 7.4378 ms]
|
|
29
|
+
thrpt: [2.1008 GiB/s 2.1252 GiB/s 2.1493 GiB/s]
|
|
30
|
+
s3/read_half/1c741003-40ef-43a9-b23f-b6a32ed7c4c6
|
|
31
|
+
time: [7.0645 ms 7.1524 ms 7.2473 ms]
|
|
32
|
+
thrpt: [1.0780 GiB/s 1.0923 GiB/s 1.1059 GiB/s]
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
So our current behavior is buggy, and we need more clear API to address that.
|
|
36
|
+
|
|
37
|
+
# Guide-level explanation
|
|
38
|
+
|
|
39
|
+
We will remove `Reader::total_size()` from public API instead of adding the following APIs for `Object`:
|
|
40
|
+
|
|
41
|
+
```rust
|
|
42
|
+
pub fn reader(&self) -> Reader {}
|
|
43
|
+
pub fn range_reader(&self, offset: u64, size: u64) -> Reader {}
|
|
44
|
+
pub fn offset_reader(&self, offset: u64) -> Reader {}
|
|
45
|
+
pub fn limited_reader(&self, size: u64) -> Reader {}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
- `reader`: returns a new reader who can read the whole file.
|
|
49
|
+
- `range_reader`: returns a ranged reader which read `[offset, offset+size)`.
|
|
50
|
+
- `offset_reader`: returns a reader from offset `[offset:]`
|
|
51
|
+
- `limited_reader`: returns a limited reader `[:size]`
|
|
52
|
+
|
|
53
|
+
Take `parquet`'s actual logic as an example. We can rewrite:
|
|
54
|
+
|
|
55
|
+
```rust
|
|
56
|
+
async fn _read_single_column_async<'b, R, F>(
|
|
57
|
+
factory: F,
|
|
58
|
+
meta: &ColumnChunkMetaData,
|
|
59
|
+
) -> Result<(&ColumnChunkMetaData, Vec<u8>)>
|
|
60
|
+
where
|
|
61
|
+
R: AsyncRead + AsyncSeek + Send + Unpin,
|
|
62
|
+
F: Fn() -> BoxFuture<'b, std::io::Result<R>>,
|
|
63
|
+
{
|
|
64
|
+
let mut reader = factory().await?;
|
|
65
|
+
let (start, len) = meta.byte_range();
|
|
66
|
+
reader.seek(std::io::SeekFrom::Start(start)).await?;
|
|
67
|
+
let mut chunk = vec![0; len as usize];
|
|
68
|
+
reader.read_exact(&mut chunk).await?;
|
|
69
|
+
Result::Ok((meta, chunk))
|
|
70
|
+
}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
into
|
|
74
|
+
|
|
75
|
+
```rust
|
|
76
|
+
async fn _read_single_column_async<'b, R, F>(
|
|
77
|
+
factory: F,
|
|
78
|
+
meta: &ColumnChunkMetaData,
|
|
79
|
+
) -> Result<(&ColumnChunkMetaData, Vec<u8>)>
|
|
80
|
+
where
|
|
81
|
+
R: AsyncRead + AsyncSeek + Send + Unpin,
|
|
82
|
+
F: Fn(usize, usize) -> BoxFuture<'b, std::io::Result<R>>,
|
|
83
|
+
{
|
|
84
|
+
let (start, len) = meta.byte_range();
|
|
85
|
+
let mut reader = factory(start, len).await?;
|
|
86
|
+
let mut chunk = vec![0; len as usize];
|
|
87
|
+
reader.read_exact(&mut chunk).await?;
|
|
88
|
+
Result::Ok((meta, chunk))
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
So that:
|
|
93
|
+
|
|
94
|
+
- No extra data will be read.
|
|
95
|
+
- No extra `seek`/`stat` operation is needed.
|
|
96
|
+
|
|
97
|
+
# Reference-level explanation
|
|
98
|
+
|
|
99
|
+
Inside `Reader`, we will correctly maintain `offset`, `size`, and `pos`.
|
|
100
|
+
|
|
101
|
+
- If `offset` is `None`, we will use `0` instead.
|
|
102
|
+
- If `size` is `None`, we will use `meta.content_length() - self.offset.unwrap_or_default()` instead.
|
|
103
|
+
|
|
104
|
+
We will calculate `Reader` current offset and size easily:
|
|
105
|
+
|
|
106
|
+
```rust
|
|
107
|
+
fn current_offset(&self) -> u64 {
|
|
108
|
+
self.offset.unwrap_or_default() + self.pos
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
fn current_size(&self) -> Option<u64> {
|
|
112
|
+
self.size.map(|v| v - self.pos)
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Instead of constantly requesting the entire object content, we will set the size:
|
|
117
|
+
|
|
118
|
+
```rust
|
|
119
|
+
let op = OpRead {
|
|
120
|
+
path: self.path.to_string(),
|
|
121
|
+
offset: Some(self.current_offset()),
|
|
122
|
+
size: self.current_size(),
|
|
123
|
+
};
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
After this change, we will have a similar throughput for `read_all` and `read_half`:
|
|
127
|
+
|
|
128
|
+
```txt
|
|
129
|
+
s3/read/6dd40f8d-7455-451e-b510-3b7ac23e0468
|
|
130
|
+
time: [4.9554 ms 5.0888 ms 5.2282 ms]
|
|
131
|
+
thrpt: [2.9886 GiB/s 3.0704 GiB/s 3.1532 GiB/s]
|
|
132
|
+
s3/read_half/6dd40f8d-7455-451e-b510-3b7ac23e0468
|
|
133
|
+
time: [3.1868 ms 3.2494 ms 3.3052 ms]
|
|
134
|
+
thrpt: [2.3637 GiB/s 2.4043 GiB/s 2.4515 GiB/s]
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
# Drawbacks
|
|
138
|
+
|
|
139
|
+
None
|
|
140
|
+
|
|
141
|
+
# Rationale and alternatives
|
|
142
|
+
|
|
143
|
+
None
|
|
144
|
+
|
|
145
|
+
# Prior art
|
|
146
|
+
|
|
147
|
+
None
|
|
148
|
+
|
|
149
|
+
# Unresolved questions
|
|
150
|
+
|
|
151
|
+
None
|
|
152
|
+
|
|
153
|
+
# Future possibilities
|
|
154
|
+
|
|
155
|
+
- Refactor the parquet reading logic to make the most use of `range_reader`.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
- Proposal Name: `path-normalization`
|
|
2
|
+
- Start Date: 2022-03-08
|
|
3
|
+
- RFC PR: [apache/opendal#112](https://github.com/apache/opendal/pull/112)
|
|
4
|
+
- Tracking Issue: [apache/opendal#112](https://github.com/apache/opendal/issues/112)
|
|
5
|
+
|
|
6
|
+
# Summary
|
|
7
|
+
|
|
8
|
+
Implement path normalization to enhance user experience.
|
|
9
|
+
|
|
10
|
+
# Motivation
|
|
11
|
+
|
|
12
|
+
OpenDAL's current path behavior makes users confused:
|
|
13
|
+
|
|
14
|
+
- [operator.object("/admin/data/") error](https://github.com/apache/opendal/issues/107)
|
|
15
|
+
- [Read /admin/data//ontime_200.csv return empty](https://github.com/apache/opendal/issues/109)
|
|
16
|
+
|
|
17
|
+
They are different bugs that reflect the exact root cause: the path is not well normalized.
|
|
18
|
+
|
|
19
|
+
On local fs, we can read the same path with different path: `abc/def/../def`, `abc/def`, `abc//def`, `abc/./def`.
|
|
20
|
+
|
|
21
|
+
There is no magic here: our stdlib does the dirty job. For example:
|
|
22
|
+
|
|
23
|
+
- [std::path::PathBuf::canonicalize](https://doc.rust-lang.org/std/path/struct.PathBuf.html#method.canonicalize): Returns the canonical, absolute form of the path with all intermediate components normalized and symbolic links resolved.
|
|
24
|
+
- [std::path::PathBuf::components](https://doc.rust-lang.org/std/path/struct.PathBuf.html#method.components): Produces an iterator over the Components of the path. When parsing the path, there is a small amount of normalization...
|
|
25
|
+
|
|
26
|
+
But for s3 alike storage system, there's no such helpers: `abc/def/../def`, `abc/def`, `abc//def`, `abc/./def` refers entirely different objects. So users may confuse why I can't get the object with this path.
|
|
27
|
+
|
|
28
|
+
So OpenDAL needs to implement path normalization to enhance the user experience.
|
|
29
|
+
|
|
30
|
+
# Guide-level explanation
|
|
31
|
+
|
|
32
|
+
We will do path normalization automatically.
|
|
33
|
+
|
|
34
|
+
The following rules will be applied (so far):
|
|
35
|
+
|
|
36
|
+
- Remove `//` inside path: `op.object("abc/def")` and `op.object("abc//def")` will resolve to the same object.
|
|
37
|
+
- Make sure path under `root`: `op.object("/abc")` and `op.object("abc")` will resolve to the same object.
|
|
38
|
+
|
|
39
|
+
Other rules still need more consideration to leave them for the future.
|
|
40
|
+
|
|
41
|
+
# Reference-level explanation
|
|
42
|
+
|
|
43
|
+
We will build the absolute path via `{root}/{path}` and replace all `//` into `/` instead.
|
|
44
|
+
|
|
45
|
+
# Drawbacks
|
|
46
|
+
|
|
47
|
+
None
|
|
48
|
+
|
|
49
|
+
# Rationale and alternatives
|
|
50
|
+
|
|
51
|
+
## How about the link?
|
|
52
|
+
|
|
53
|
+
If we build an actual path via `{root}/{path}`, the link object may be inaccessible.
|
|
54
|
+
|
|
55
|
+
I don't have good ideas so far. Maybe we can add a new flag to control the link behavior. For now, there's no feature request for link support.
|
|
56
|
+
|
|
57
|
+
Let's leave for the future to resolve.
|
|
58
|
+
|
|
59
|
+
## S3 URI Clean
|
|
60
|
+
|
|
61
|
+
For s3, `abc//def` is different from `abc/def` indeed. To make it possible to access not normalized path, we can provide a new flag for the builder:
|
|
62
|
+
|
|
63
|
+
```rust
|
|
64
|
+
let builder = Backend::build().disable_path_normalization()
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
In this way, the user can control the path more precisely.
|
|
68
|
+
|
|
69
|
+
# Prior art
|
|
70
|
+
|
|
71
|
+
None
|
|
72
|
+
|
|
73
|
+
# Unresolved questions
|
|
74
|
+
|
|
75
|
+
None
|
|
76
|
+
|
|
77
|
+
# Future possibilities
|
|
78
|
+
|
|
79
|
+
None
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
- Proposal Name: `async_streaming_io`
|
|
2
|
+
- Start Date: 2022-03-28
|
|
3
|
+
- RFC PR: [apache/opendal#191](https://github.com/apache/opendal/pull/191)
|
|
4
|
+
- Tracking Issue: [apache/opendal#190](https://github.com/apache/opendal/issues/190)
|
|
5
|
+
|
|
6
|
+
**Reverted**
|
|
7
|
+
|
|
8
|
+
# Summary
|
|
9
|
+
|
|
10
|
+
Use `Stream`/`Sink` instead of `AsyncRead` in `Accessor`.
|
|
11
|
+
|
|
12
|
+
# Motivation
|
|
13
|
+
|
|
14
|
+
`Accessor` intends to be the `underlying trait of all backends for implementers`. However, it's not so underlying enough.
|
|
15
|
+
|
|
16
|
+
## Over-wrapped
|
|
17
|
+
|
|
18
|
+
`Accessor` returns a `BoxedAsyncReader` for `read` operation:
|
|
19
|
+
|
|
20
|
+
```rust
|
|
21
|
+
pub type BoxedAsyncReader = Box<dyn AsyncRead + Unpin + Send>;
|
|
22
|
+
|
|
23
|
+
pub trait Accessor {
|
|
24
|
+
async fn read(&self, args: &OpRead) -> Result<BoxedAsyncReader> {
|
|
25
|
+
let _ = args;
|
|
26
|
+
unimplemented!()
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
And we are exposing `Reader`, which implements `AsyncRead` and `AsyncSeek` to end-users. For every call to `Reader::poll_read()`, we need:
|
|
32
|
+
|
|
33
|
+
- `Reader::poll_read()`
|
|
34
|
+
- `BoxedAsyncReader::poll_read()`
|
|
35
|
+
- `IntoAsyncRead<ByteStream>::poll_read()`
|
|
36
|
+
- `ByteStream::poll_next()`
|
|
37
|
+
|
|
38
|
+
If we could return a `Stream` directly, we can transform the call stack into:
|
|
39
|
+
|
|
40
|
+
- `Reader::poll_read()`
|
|
41
|
+
- `ByteStream::poll_next()`
|
|
42
|
+
|
|
43
|
+
In this way, we operate on the underlying IO stream, and the caller must keep track of the reading states.
|
|
44
|
+
|
|
45
|
+
## Inconsistent
|
|
46
|
+
|
|
47
|
+
OpenDAL's `read` and `write` behavior is not consistent.
|
|
48
|
+
|
|
49
|
+
```rust
|
|
50
|
+
pub type BoxedAsyncReader = Box<dyn AsyncRead + Unpin + Send>;
|
|
51
|
+
|
|
52
|
+
pub trait Accessor: Send + Sync + Debug {
|
|
53
|
+
async fn read(&self, args: &OpRead) -> Result<BoxedAsyncReader> {
|
|
54
|
+
let _ = args;
|
|
55
|
+
unimplemented!()
|
|
56
|
+
}
|
|
57
|
+
async fn write(&self, r: BoxedAsyncReader, args: &OpWrite) -> Result<usize> {
|
|
58
|
+
let (_, _) = (r, args);
|
|
59
|
+
unimplemented!()
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
For `read`, OpenDAL returns a `BoxedAsyncReader` which users can decide when and how to read data. But for `write`, OpenDAL accepts a `BoxedAsyncReader` instead, in which users can't control the writing logic. How large will the writing buffer size be? When to call `flush`?
|
|
65
|
+
|
|
66
|
+
## Service native optimization
|
|
67
|
+
|
|
68
|
+
OpenDAL knows more about the service detail, but returning `BoxedAsyncReader` makes it can't fully use the advantage.
|
|
69
|
+
|
|
70
|
+
For example, most object storage services use HTTP to transfer data which is TCP stream-based. The most efficient way is to return a full TCP buffer, but users don't know about that. First, users could have continuous small reads on stream. To overcome the poor performance, they have to use `BufReader`, which adds a new buffering between reading. Then, users don't know the correct (best) buffer size to set.
|
|
71
|
+
|
|
72
|
+
Via returning a `Stream`, users could benefit from it in both ways:
|
|
73
|
+
|
|
74
|
+
- Users who want underlying control can operate on the `Stream` directly.
|
|
75
|
+
- Users who don't care about the behavior can use OpenDAL provided Reader, which always adopts the best optimization.
|
|
76
|
+
|
|
77
|
+
# Guide-level explanation
|
|
78
|
+
|
|
79
|
+
Within the `async_streaming_io` feature, we will add the following new APIs to `Object`:
|
|
80
|
+
|
|
81
|
+
```rust
|
|
82
|
+
impl Object {
|
|
83
|
+
pub async fn stream(&self, offset: Option<u64>, size: Option<u64>) -> Result<BytesStream> {}
|
|
84
|
+
pub async fn sink(&self, size: u64) -> Result<BytesSink> {}
|
|
85
|
+
}
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Users can control the underlying logic of those bytes, streams, and sinks.
|
|
89
|
+
|
|
90
|
+
For example, they can:
|
|
91
|
+
|
|
92
|
+
- Read data on demand: `stream.next().await`
|
|
93
|
+
- Write data on demand: `sink.feed(bs).await; sink.close().await;`
|
|
94
|
+
|
|
95
|
+
Based on `stream` and `sink`, `Object` will provide more optimized helper functions like:
|
|
96
|
+
|
|
97
|
+
- `async read(offset: Option<u64>, size: Option<u64>) -> Result<bytes::Bytes>`
|
|
98
|
+
- `async write(bs: bytes::Bytes) -> Result<()>`
|
|
99
|
+
|
|
100
|
+
# Reference-level explanation
|
|
101
|
+
|
|
102
|
+
`read` and `write` in `Accessor` will be refactored into streaming-based:
|
|
103
|
+
|
|
104
|
+
```rust
|
|
105
|
+
pub type BytesStream = Box<dyn Stream + Unpin + Send>;
|
|
106
|
+
pub type BytesSink = Box<dyn Sink + Unpin + Send>;
|
|
107
|
+
|
|
108
|
+
pub trait Accessor: Send + Sync + Debug {
|
|
109
|
+
async fn read(&self, args: &OpRead) -> Result<BytesStream> {
|
|
110
|
+
let _ = args;
|
|
111
|
+
unimplemented!()
|
|
112
|
+
}
|
|
113
|
+
async fn write(&self, args: &OpWrite) -> Result<BytesSink> {
|
|
114
|
+
let _ = args;
|
|
115
|
+
unimplemented!()
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
All other IO functions will be adapted to fit these changes.
|
|
121
|
+
|
|
122
|
+
For fs, it's simple to implement `Stream` and `Sink` for `tokio::fs::File`.
|
|
123
|
+
|
|
124
|
+
We will return a `BodySinker` instead for all HTTP-based storage services. In which we maintain a `put_object` `ResponseFuture` that construct by `hyper` and a `sender` part of the channel. All data sent by users will be passed to `ResponseFuture` via the unbuffered channel.
|
|
125
|
+
|
|
126
|
+
```rust
|
|
127
|
+
struct BodySinker {
|
|
128
|
+
fut: ResponseFuture,
|
|
129
|
+
sender: Sender<bytes::Bytes>
|
|
130
|
+
}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
# Drawbacks
|
|
134
|
+
|
|
135
|
+
## Performance regression on fs
|
|
136
|
+
|
|
137
|
+
`fs` is not stream based backend, and convert from `Reader` to `Stream` is not zero cost. Based on benchmark over `IntoStream`, we can get nearly 70% performance drawback (pure memory):
|
|
138
|
+
|
|
139
|
+
```rust
|
|
140
|
+
into_stream/into_stream time: [1.3046 ms 1.3056 ms 1.3068 ms]
|
|
141
|
+
thrpt: [2.9891 GiB/s 2.9919 GiB/s 2.9942 GiB/s]
|
|
142
|
+
into_stream/raw_reader time: [382.10 us 383.52 us 385.16 us]
|
|
143
|
+
thrpt: [10.142 GiB/s 10.185 GiB/s 10.223 GiB/s]
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
However, real fs is not as fast as memory and most overhead will happen at disk side, so that performance regression is allowed (at least at this time).
|
|
147
|
+
|
|
148
|
+
# Rationale and alternatives
|
|
149
|
+
|
|
150
|
+
## Performance for switching from Reader to Stream
|
|
151
|
+
|
|
152
|
+
Before
|
|
153
|
+
|
|
154
|
+
```rust
|
|
155
|
+
read_full/4.00 KiB time: [455.70 us 466.18 us 476.93 us]
|
|
156
|
+
thrpt: [8.1904 MiB/s 8.3794 MiB/s 8.5719 MiB/s]
|
|
157
|
+
read_full/256 KiB time: [530.63 us 544.30 us 557.84 us]
|
|
158
|
+
thrpt: [448.16 MiB/s 459.30 MiB/s 471.14 MiB/s]
|
|
159
|
+
read_full/4.00 MiB time: [1.5569 ms 1.6152 ms 1.6743 ms]
|
|
160
|
+
thrpt: [2.3330 GiB/s 2.4184 GiB/s 2.5090 GiB/s]
|
|
161
|
+
read_full/16.0 MiB time: [5.7337 ms 5.9087 ms 6.0813 ms]
|
|
162
|
+
thrpt: [2.5693 GiB/s 2.6444 GiB/s 2.7251 GiB/s]
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
After
|
|
166
|
+
|
|
167
|
+
```rust
|
|
168
|
+
read_full/4.00 KiB time: [455.67 us 466.03 us 476.21 us]
|
|
169
|
+
thrpt: [8.2027 MiB/s 8.3819 MiB/s 8.5725 MiB/s]
|
|
170
|
+
change:
|
|
171
|
+
time: [-2.1168% +0.6241% +3.8735%] (p = 0.68 > 0.05)
|
|
172
|
+
thrpt: [-3.7291% -0.6203% +2.1625%]
|
|
173
|
+
No change in performance detected.
|
|
174
|
+
read_full/256 KiB time: [521.04 us 535.20 us 548.74 us]
|
|
175
|
+
thrpt: [455.59 MiB/s 467.11 MiB/s 479.81 MiB/s]
|
|
176
|
+
change:
|
|
177
|
+
time: [-7.8470% -4.7987% -1.4955%] (p = 0.01 < 0.05)
|
|
178
|
+
thrpt: [+1.5182% +5.0406% +8.5152%]
|
|
179
|
+
Performance has improved.
|
|
180
|
+
read_full/4.00 MiB time: [1.4571 ms 1.5184 ms 1.5843 ms]
|
|
181
|
+
thrpt: [2.4655 GiB/s 2.5725 GiB/s 2.6808 GiB/s]
|
|
182
|
+
change:
|
|
183
|
+
time: [-5.4403% -1.5696% +2.3719%] (p = 0.44 > 0.05)
|
|
184
|
+
thrpt: [-2.3170% +1.5946% +5.7533%]
|
|
185
|
+
No change in performance detected.
|
|
186
|
+
read_full/16.0 MiB time: [5.0201 ms 5.2105 ms 5.3986 ms]
|
|
187
|
+
thrpt: [2.8943 GiB/s 2.9988 GiB/s 3.1125 GiB/s]
|
|
188
|
+
change:
|
|
189
|
+
time: [-15.917% -11.816% -7.5219%] (p = 0.00 < 0.05)
|
|
190
|
+
thrpt: [+8.1337% +13.400% +18.930%]
|
|
191
|
+
Performance has improved.
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Performance for the extra channel in `write`
|
|
195
|
+
|
|
196
|
+
Based on the benchmark during research, the **unbuffered** channel does improve the performance a bit in some cases:
|
|
197
|
+
|
|
198
|
+
Before:
|
|
199
|
+
|
|
200
|
+
```rust
|
|
201
|
+
write_once/4.00 KiB time: [564.11 us 575.17 us 586.15 us]
|
|
202
|
+
thrpt: [6.6642 MiB/s 6.7914 MiB/s 6.9246 MiB/s]
|
|
203
|
+
write_once/256 KiB time: [1.3600 ms 1.3896 ms 1.4168 ms]
|
|
204
|
+
thrpt: [176.46 MiB/s 179.90 MiB/s 183.82 MiB/s]
|
|
205
|
+
write_once/4.00 MiB time: [11.394 ms 11.555 ms 11.717 ms]
|
|
206
|
+
thrpt: [341.39 MiB/s 346.18 MiB/s 351.07 MiB/s]
|
|
207
|
+
write_once/16.0 MiB time: [41.829 ms 42.645 ms 43.454 ms]
|
|
208
|
+
thrpt: [368.20 MiB/s 375.19 MiB/s 382.51 MiB/s]
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
After:
|
|
212
|
+
|
|
213
|
+
```rust
|
|
214
|
+
write_once/4.00 KiB time: [572.20 us 583.62 us 595.21 us]
|
|
215
|
+
thrpt: [6.5628 MiB/s 6.6932 MiB/s 6.8267 MiB/s]
|
|
216
|
+
change:
|
|
217
|
+
time: [-6.3126% -3.8179% -1.0733%] (p = 0.00 < 0.05)
|
|
218
|
+
thrpt: [+1.0849% +3.9695% +6.7380%]
|
|
219
|
+
Performance has improved.
|
|
220
|
+
write_once/256 KiB time: [1.3192 ms 1.3456 ms 1.3738 ms]
|
|
221
|
+
thrpt: [181.98 MiB/s 185.79 MiB/s 189.50 MiB/s]
|
|
222
|
+
change:
|
|
223
|
+
time: [-0.5899% +1.7476% +4.1037%] (p = 0.15 > 0.05)
|
|
224
|
+
thrpt: [-3.9420% -1.7176% +0.5934%]
|
|
225
|
+
No change in performance detected.
|
|
226
|
+
write_once/4.00 MiB time: [10.855 ms 11.039 ms 11.228 ms]
|
|
227
|
+
thrpt: [356.25 MiB/s 362.34 MiB/s 368.51 MiB/s]
|
|
228
|
+
change:
|
|
229
|
+
time: [-6.9651% -4.8176% -2.5681%] (p = 0.00 < 0.05)
|
|
230
|
+
thrpt: [+2.6358% +5.0614% +7.4866%]
|
|
231
|
+
Performance has improved.
|
|
232
|
+
write_once/16.0 MiB time: [38.706 ms 39.577 ms 40.457 ms]
|
|
233
|
+
thrpt: [395.48 MiB/s 404.27 MiB/s 413.37 MiB/s]
|
|
234
|
+
change:
|
|
235
|
+
time: [-10.829% -8.3611% -5.8702%] (p = 0.00 < 0.05)
|
|
236
|
+
thrpt: [+6.2363% +9.1240% +12.145%]
|
|
237
|
+
Performance has improved.
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
## Add complexity on the services side
|
|
241
|
+
|
|
242
|
+
Returning `Stream` and `Sink` make it complex to implement. At first glance, it does. But in reality, it's not.
|
|
243
|
+
|
|
244
|
+
Note: HTTP (especially for hyper) is stream-oriented.
|
|
245
|
+
|
|
246
|
+
- Returning a `stream` is more straightforward than `reader`.
|
|
247
|
+
- Returning `Sink` is covered by the global shared `BodySinker` struct.
|
|
248
|
+
|
|
249
|
+
Other helper functions will be covered at the Object-level which services don't need to bother.
|
|
250
|
+
|
|
251
|
+
# Prior art
|
|
252
|
+
|
|
253
|
+
## Returning a `Writer`
|
|
254
|
+
|
|
255
|
+
The most natural extending is to return `BoxedAsyncWriter`:
|
|
256
|
+
|
|
257
|
+
```rust
|
|
258
|
+
pub trait Accessor: Send + Sync + Debug {
|
|
259
|
+
/// Read data from the underlying storage into input writer.
|
|
260
|
+
async fn read(&self, args: &OpRead) -> Result<BoxedAsyncReader> {
|
|
261
|
+
let _ = args;
|
|
262
|
+
unimplemented!()
|
|
263
|
+
}
|
|
264
|
+
/// Write data from input reader to the underlying storage.
|
|
265
|
+
async fn write(&self, args: &OpWrite) -> Result<BoxedAsyncWriter> {
|
|
266
|
+
let _ = args;
|
|
267
|
+
unimplemented!()
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
But it only fixes the `Inconsistent` concern and can't help with other issues.
|
|
273
|
+
|
|
274
|
+
## Slice based API
|
|
275
|
+
|
|
276
|
+
Most rust IO APIs are based on slice:
|
|
277
|
+
|
|
278
|
+
```rust
|
|
279
|
+
pub trait Accessor: Send + Sync + Debug {
|
|
280
|
+
/// Read data from the underlying storage into input writer.
|
|
281
|
+
async fn read(&self, args: &OpRead, bs: &mut [u8]) -> Result<usize> {
|
|
282
|
+
let _ = args;
|
|
283
|
+
unimplemented!()
|
|
284
|
+
}
|
|
285
|
+
/// Write data from input reader to the underlying storage.
|
|
286
|
+
async fn write(&self, args: &OpWrite, bs: &[u8]) -> Result<usize> {
|
|
287
|
+
let _ = args;
|
|
288
|
+
unimplemented!()
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
The problem is `Accessor` doesn't have states:
|
|
294
|
+
|
|
295
|
+
- If we require all data must be passed at one time, we can't support large files read & write
|
|
296
|
+
- If we allow users to call `read`/`write` multiple times, we need to implement another `Reader` and `Writer` alike logic.
|
|
297
|
+
|
|
298
|
+
## Accept `Reader` and `Writer`
|
|
299
|
+
|
|
300
|
+
It's also possible to accept `Reader` and `Writer` instead.
|
|
301
|
+
|
|
302
|
+
```rust
|
|
303
|
+
pub trait Accessor: Send + Sync + Debug {
|
|
304
|
+
/// Read data from the underlying storage into input writer.
|
|
305
|
+
async fn read(&self, args: &OpRead, w: BoxedAsyncWriter) -> Result<usize> {
|
|
306
|
+
let _ = args;
|
|
307
|
+
unimplemented!()
|
|
308
|
+
}
|
|
309
|
+
/// Write data from input reader to the underlying storage.
|
|
310
|
+
async fn write(&self, args: &OpWrite, r: BoxedAsyncReader) -> Result<usize> {
|
|
311
|
+
let _ = args;
|
|
312
|
+
unimplemented!()
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
This API design addressed all concerns but made it hard for users to use. Primarily, we can't support `futures::AsyncRead` and `tokio::AsyncRead` simultaneously.
|
|
318
|
+
|
|
319
|
+
For example, we can't accept a `Box::new(Vec::new())`, user can't get this vec from OpenDAL.
|
|
320
|
+
|
|
321
|
+
# Unresolved questions
|
|
322
|
+
|
|
323
|
+
None.
|
|
324
|
+
|
|
325
|
+
# Future possibilities
|
|
326
|
+
|
|
327
|
+
- Implement `Object::read_into(w: BoxedAsyncWriter)`
|
|
328
|
+
- Implement `Object::write_from(r: BoxedAsyncReader)`
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
- Proposal Name: `remove_credential`
|
|
2
|
+
- Start Date: 2022-04-02
|
|
3
|
+
- RFC PR: [apache/opendal#203](https://github.com/apache/opendal/pull/203)
|
|
4
|
+
- Tracking Issue: [apache/opendal#203](https://github.com/apache/opendal/issues/203)
|
|
5
|
+
|
|
6
|
+
# Summary
|
|
7
|
+
|
|
8
|
+
Remove the concept of credential.
|
|
9
|
+
|
|
10
|
+
# Motivation
|
|
11
|
+
|
|
12
|
+
`Credential` intends to carry service credentials like `access_key_id` and `secret_access_key`. At OpenDAL, we designed a global `Credential` enum for services and users to use.
|
|
13
|
+
|
|
14
|
+
```rust
|
|
15
|
+
pub enum Credential {
|
|
16
|
+
/// Plain refers to no credential has been provided, fallback to services'
|
|
17
|
+
/// default logic.
|
|
18
|
+
Plain,
|
|
19
|
+
/// Basic refers to HTTP Basic Authentication.
|
|
20
|
+
Basic { username: String, password: String },
|
|
21
|
+
/// HMAC, also known as Access Key/Secret Key authentication.
|
|
22
|
+
HMAC {
|
|
23
|
+
access_key_id: String,
|
|
24
|
+
secret_access_key: String,
|
|
25
|
+
},
|
|
26
|
+
/// Token refers to static API token.
|
|
27
|
+
Token(String),
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
However, every service only supports one kind of `Credential` with different `Credential` load methods covered by [reqsign](https://github.com/Xuanwo/reqsign). As a result, only `HMAC` is used. Both users and services need to write the same logic again and again.
|
|
32
|
+
|
|
33
|
+
# Guide-level explanation
|
|
34
|
+
|
|
35
|
+
`Credential` will be removed, and the services builder will provide native credential representation directly.
|
|
36
|
+
|
|
37
|
+
For s3:
|
|
38
|
+
|
|
39
|
+
```rust
|
|
40
|
+
pub fn access_key_id(&mut self, v: &str) -> &mut Self {}
|
|
41
|
+
pub fn secret_access_key(&mut self, v: &str) -> &mut Self {}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
For azblob:
|
|
45
|
+
|
|
46
|
+
```rust
|
|
47
|
+
pub fn account_name(&mut self, account_name: &str) -> &mut Self {}
|
|
48
|
+
pub fn account_key(&mut self, account_key: &str) -> &mut Self {}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
All builders must implement `Debug` by hand and redact sensitive fields to avoid credentials being a leak.
|
|
52
|
+
|
|
53
|
+
```rust
|
|
54
|
+
impl Debug for Builder {
|
|
55
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
56
|
+
let mut ds = f.debug_struct("Builder");
|
|
57
|
+
|
|
58
|
+
ds.field("root", &self.root);
|
|
59
|
+
ds.field("container", &self.container);
|
|
60
|
+
ds.field("endpoint", &self.endpoint);
|
|
61
|
+
|
|
62
|
+
if self.account_name.is_some() {
|
|
63
|
+
ds.field("account_name", &"<redacted>");
|
|
64
|
+
}
|
|
65
|
+
if self.account_key.is_some() {
|
|
66
|
+
ds.field("account_key", &"<redacted>");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
ds.finish_non_exhaustive()
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
# Reference-level explanation
|
|
75
|
+
|
|
76
|
+
Simple change without reference-level explanation needs.
|
|
77
|
+
|
|
78
|
+
# Drawbacks
|
|
79
|
+
|
|
80
|
+
API Breakage.
|
|
81
|
+
|
|
82
|
+
# Rationale and alternatives
|
|
83
|
+
|
|
84
|
+
None
|
|
85
|
+
|
|
86
|
+
# Prior art
|
|
87
|
+
|
|
88
|
+
None
|
|
89
|
+
|
|
90
|
+
# Unresolved questions
|
|
91
|
+
|
|
92
|
+
None
|
|
93
|
+
|
|
94
|
+
# Future possibilities
|
|
95
|
+
|
|
96
|
+
None
|