opendal 0.1.6.pre.rc.1-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. checksums.yaml +7 -0
  2. data/.standard.yml +20 -0
  3. data/.tool-versions +1 -0
  4. data/.yardopts +1 -0
  5. data/DEPENDENCIES.md +9 -0
  6. data/DEPENDENCIES.rust.tsv +277 -0
  7. data/Gemfile +35 -0
  8. data/README.md +159 -0
  9. data/Rakefile +149 -0
  10. data/core/CHANGELOG.md +4929 -0
  11. data/core/CONTRIBUTING.md +61 -0
  12. data/core/DEPENDENCIES.md +3 -0
  13. data/core/DEPENDENCIES.rust.tsv +185 -0
  14. data/core/LICENSE +201 -0
  15. data/core/README.md +228 -0
  16. data/core/benches/README.md +18 -0
  17. data/core/benches/ops/README.md +26 -0
  18. data/core/benches/types/README.md +9 -0
  19. data/core/benches/vs_fs/README.md +35 -0
  20. data/core/benches/vs_s3/README.md +55 -0
  21. data/core/edge/README.md +3 -0
  22. data/core/edge/file_write_on_full_disk/README.md +14 -0
  23. data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
  24. data/core/edge/s3_read_on_wasm/.gitignore +3 -0
  25. data/core/edge/s3_read_on_wasm/README.md +42 -0
  26. data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
  27. data/core/examples/README.md +23 -0
  28. data/core/examples/basic/README.md +15 -0
  29. data/core/examples/concurrent-upload/README.md +15 -0
  30. data/core/examples/multipart-upload/README.md +15 -0
  31. data/core/fuzz/.gitignore +5 -0
  32. data/core/fuzz/README.md +68 -0
  33. data/core/src/docs/comparisons/vs_object_store.md +183 -0
  34. data/core/src/docs/performance/concurrent_write.md +101 -0
  35. data/core/src/docs/performance/http_optimization.md +124 -0
  36. data/core/src/docs/rfcs/0000_example.md +74 -0
  37. data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
  38. data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
  39. data/core/src/docs/rfcs/0044_error_handle.md +198 -0
  40. data/core/src/docs/rfcs/0057_auto_region.md +160 -0
  41. data/core/src/docs/rfcs/0069_object_stream.md +145 -0
  42. data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
  43. data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
  44. data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
  45. data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
  46. data/core/src/docs/rfcs/0221_create_dir.md +89 -0
  47. data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
  48. data/core/src/docs/rfcs/0293_object_id.md +67 -0
  49. data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
  50. data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
  51. data/core/src/docs/rfcs/0413_presign.md +154 -0
  52. data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
  53. data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
  54. data/core/src/docs/rfcs/0438_multipart.md +163 -0
  55. data/core/src/docs/rfcs/0443_gateway.md +73 -0
  56. data/core/src/docs/rfcs/0501_new_builder.md +111 -0
  57. data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
  58. data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
  59. data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
  60. data/core/src/docs/rfcs/0623_redis_service.md +300 -0
  61. data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
  62. data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
  63. data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
  64. data/core/src/docs/rfcs/0926_object_reader.md +93 -0
  65. data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
  66. data/core/src/docs/rfcs/1085_object_handler.md +73 -0
  67. data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
  68. data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
  69. data/core/src/docs/rfcs/1420_object_writer.md +147 -0
  70. data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
  71. data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
  72. data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
  73. data/core/src/docs/rfcs/2133_append_api.md +88 -0
  74. data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
  75. data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
  76. data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
  77. data/core/src/docs/rfcs/2774_lister_api.md +66 -0
  78. data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
  79. data/core/src/docs/rfcs/2852_native_capability.md +58 -0
  80. data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
  81. data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
  82. data/core/src/docs/rfcs/3197_config.md +237 -0
  83. data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
  84. data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
  85. data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
  86. data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
  87. data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
  88. data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
  89. data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
  90. data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
  91. data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
  92. data/core/src/docs/rfcs/4638_executor.md +215 -0
  93. data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
  94. data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
  95. data/core/src/docs/rfcs/5479_context.md +140 -0
  96. data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
  97. data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
  98. data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
  99. data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
  100. data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
  101. data/core/src/docs/rfcs/6209_glob_support.md +132 -0
  102. data/core/src/docs/rfcs/6213_options_api.md +142 -0
  103. data/core/src/docs/rfcs/README.md +62 -0
  104. data/core/src/docs/upgrade.md +1556 -0
  105. data/core/src/services/aliyun_drive/docs.md +61 -0
  106. data/core/src/services/alluxio/docs.md +45 -0
  107. data/core/src/services/azblob/docs.md +77 -0
  108. data/core/src/services/azdls/docs.md +73 -0
  109. data/core/src/services/azfile/docs.md +65 -0
  110. data/core/src/services/b2/docs.md +54 -0
  111. data/core/src/services/cacache/docs.md +38 -0
  112. data/core/src/services/cloudflare_kv/docs.md +21 -0
  113. data/core/src/services/cos/docs.md +55 -0
  114. data/core/src/services/d1/docs.md +48 -0
  115. data/core/src/services/dashmap/docs.md +38 -0
  116. data/core/src/services/dbfs/docs.md +57 -0
  117. data/core/src/services/dropbox/docs.md +64 -0
  118. data/core/src/services/etcd/docs.md +45 -0
  119. data/core/src/services/foundationdb/docs.md +42 -0
  120. data/core/src/services/fs/docs.md +49 -0
  121. data/core/src/services/ftp/docs.md +42 -0
  122. data/core/src/services/gcs/docs.md +76 -0
  123. data/core/src/services/gdrive/docs.md +65 -0
  124. data/core/src/services/ghac/docs.md +84 -0
  125. data/core/src/services/github/docs.md +52 -0
  126. data/core/src/services/gridfs/docs.md +46 -0
  127. data/core/src/services/hdfs/docs.md +140 -0
  128. data/core/src/services/hdfs_native/docs.md +35 -0
  129. data/core/src/services/http/docs.md +45 -0
  130. data/core/src/services/huggingface/docs.md +61 -0
  131. data/core/src/services/ipfs/docs.md +45 -0
  132. data/core/src/services/ipmfs/docs.md +14 -0
  133. data/core/src/services/koofr/docs.md +51 -0
  134. data/core/src/services/lakefs/docs.md +62 -0
  135. data/core/src/services/memcached/docs.md +47 -0
  136. data/core/src/services/memory/docs.md +36 -0
  137. data/core/src/services/mini_moka/docs.md +19 -0
  138. data/core/src/services/moka/docs.md +42 -0
  139. data/core/src/services/mongodb/docs.md +49 -0
  140. data/core/src/services/monoiofs/docs.md +46 -0
  141. data/core/src/services/mysql/docs.md +47 -0
  142. data/core/src/services/obs/docs.md +54 -0
  143. data/core/src/services/onedrive/docs.md +115 -0
  144. data/core/src/services/opfs/docs.md +18 -0
  145. data/core/src/services/oss/docs.md +74 -0
  146. data/core/src/services/pcloud/docs.md +51 -0
  147. data/core/src/services/persy/docs.md +43 -0
  148. data/core/src/services/postgresql/docs.md +47 -0
  149. data/core/src/services/redb/docs.md +41 -0
  150. data/core/src/services/redis/docs.md +43 -0
  151. data/core/src/services/rocksdb/docs.md +54 -0
  152. data/core/src/services/s3/compatible_services.md +126 -0
  153. data/core/src/services/s3/docs.md +244 -0
  154. data/core/src/services/seafile/docs.md +54 -0
  155. data/core/src/services/sftp/docs.md +49 -0
  156. data/core/src/services/sled/docs.md +39 -0
  157. data/core/src/services/sqlite/docs.md +46 -0
  158. data/core/src/services/surrealdb/docs.md +54 -0
  159. data/core/src/services/swift/compatible_services.md +53 -0
  160. data/core/src/services/swift/docs.md +52 -0
  161. data/core/src/services/tikv/docs.md +43 -0
  162. data/core/src/services/upyun/docs.md +51 -0
  163. data/core/src/services/vercel_artifacts/docs.md +40 -0
  164. data/core/src/services/vercel_blob/docs.md +45 -0
  165. data/core/src/services/webdav/docs.md +49 -0
  166. data/core/src/services/webhdfs/docs.md +90 -0
  167. data/core/src/services/yandex_disk/docs.md +45 -0
  168. data/core/tests/behavior/README.md +77 -0
  169. data/core/tests/data/normal_dir/.gitkeep +0 -0
  170. data/core/tests/data/normal_file.txt +1041 -0
  171. data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
  172. data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
  173. data/core/users.md +13 -0
  174. data/extconf.rb +24 -0
  175. data/lib/opendal.rb +25 -0
  176. data/lib/opendal_ruby/entry.rb +35 -0
  177. data/lib/opendal_ruby/io.rb +70 -0
  178. data/lib/opendal_ruby/metadata.rb +44 -0
  179. data/lib/opendal_ruby/opendal_ruby.so +0 -0
  180. data/lib/opendal_ruby/operator.rb +29 -0
  181. data/lib/opendal_ruby/operator_info.rb +26 -0
  182. data/opendal.gemspec +91 -0
  183. data/test/blocking_op_test.rb +112 -0
  184. data/test/capability_test.rb +42 -0
  185. data/test/io_test.rb +172 -0
  186. data/test/lister_test.rb +77 -0
  187. data/test/metadata_test.rb +78 -0
  188. data/test/middlewares_test.rb +46 -0
  189. data/test/operator_info_test.rb +35 -0
  190. data/test/test_helper.rb +36 -0
  191. metadata +240 -0
data/core/README.md ADDED
@@ -0,0 +1,228 @@
1
+ # Apache OpenDALâ„¢ Rust Core: One Layer, All Storage.
2
+
3
+ [![Build Status]][actions] [![Latest Version]][crates.io] [![Crate Downloads]][crates.io] [![chat]][discord]
4
+
5
+ [build status]: https://img.shields.io/github/actions/workflow/status/apache/opendal/ci_core.yml?branch=main
6
+ [actions]: https://github.com/apache/opendal/actions?query=branch%3Amain
7
+ [latest version]: https://img.shields.io/crates/v/opendal.svg
8
+ [crates.io]: https://crates.io/crates/opendal
9
+ [crate downloads]: https://img.shields.io/crates/d/opendal.svg
10
+ [chat]: https://img.shields.io/discord/1081052318650339399
11
+ [discord]: https://opendal.apache.org/discord
12
+
13
+ Apache OpenDALâ„¢ is an Open Data Access Layer that enables seamless interaction with diverse storage services.
14
+
15
+ <img src="https://opendal.apache.org/img/architectural.png" alt="OpenDAL Architectural" width="61.8%" />
16
+
17
+ ## Useful Links
18
+
19
+ - Documentation: [release](https://docs.rs/opendal/) | [dev](https://opendal.apache.org/docs/rust/opendal/)
20
+ - [Examples](./examples)
21
+ - [Release Notes](https://docs.rs/opendal/latest/opendal/docs/changelog/index.html)
22
+ - [Upgrade Guide](https://docs.rs/opendal/latest/opendal/docs/upgrade/index.html)
23
+ - [RFC List](https://docs.rs/opendal/latest/opendal/docs/rfcs/index.html)
24
+
25
+ ## Services
26
+
27
+ OpenDAL supports the following storage [services](https://docs.rs/opendal/latest/opendal/services/index.html):
28
+
29
+ | Type | Services |
30
+ |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|
31
+ | Standard Storage Protocols | ftp http [sftp] [webdav] |
32
+ | Object Storage Services | [azblob] [cos] [gcs] [obs] [oss] [s3] <br> [b2] [openstack_swift] [upyun] [vercel_blob] |
33
+ | File Storage Services | fs [alluxio] [azdls] [azfile] [compfs] <br> [dbfs] [gridfs] [hdfs] [hdfs_native] [ipfs] [webhdfs] |
34
+ | Consumer Cloud Storage Service | [aliyun_drive] [gdrive] [onedrive] [dropbox] [koofr] <br> [pcloud] [seafile] [yandex_disk] |
35
+ | Key-Value Storage Services | [cacache] [cloudflare_kv] [dashmap] memory [etcd] <br> [foundationdb] [persy] [redis] [rocksdb] [sled] <br> [redb] [tikv] |
36
+ | Database Storage Services | [d1] [mongodb] [mysql] [postgresql] [sqlite] [surrealdb] |
37
+ | Cache Storage Services | [ghac] [memcached] [mini_moka] [moka] [vercel_artifacts] |
38
+ | Git Based Storage Services | [huggingface] |
39
+
40
+ [sftp]: https://datatracker.ietf.org/doc/html/draft-ietf-secsh-filexfer-02
41
+ [webdav]: https://datatracker.ietf.org/doc/html/rfc4918
42
+
43
+ [azblob]: https://azure.microsoft.com/en-us/services/storage/blobs/
44
+ [cos]: https://www.tencentcloud.com/products/cos
45
+ [gcs]: https://cloud.google.com/storage
46
+ [obs]: https://www.huaweicloud.com/intl/en-us/product/obs.html
47
+ [oss]: https://www.aliyun.com/product/oss
48
+ [s3]: https://aws.amazon.com/s3/
49
+ [b2]: https://www.backblaze.com/
50
+ [openstack_swift]: https://docs.openstack.org/swift/latest/
51
+ [upyun]: https://www.upyun.com/
52
+ [vercel_blob]: https://vercel.com/docs/storage/vercel-blob
53
+
54
+ [alluxio]: https://docs.alluxio.io/os/user/stable/en/api/REST-API.html
55
+ [azdls]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage/
56
+ [azfile]: https://learn.microsoft.com/en-us/rest/api/storageservices/file-service-rest-api
57
+ [compfs]: https://github.com/compio-rs/compio/
58
+ [dbfs]: https://docs.databricks.com/en/dbfs/index.html
59
+ [gridfs]: https://www.mongodb.com/docs/manual/core/gridfs/
60
+ [hdfs]: https://hadoop.apache.org/docs/r3.3.4/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html
61
+ [hdfs_native]: https://github.com/Kimahriman/hdfs-native
62
+ [ipfs]: https://ipfs.tech/
63
+ [webhdfs]: https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html
64
+
65
+ [aliyun_drive]: https://www.aliyundrive.com/
66
+ [gdrive]: https://www.google.com/drive/
67
+ [onedrive]: https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage
68
+ [dropbox]: https://www.dropbox.com/
69
+ [koofr]: https://koofr.eu/
70
+ [pcloud]: https://www.pcloud.com/
71
+ [seafile]: https://www.seafile.com/
72
+ [yandex_disk]: https://360.yandex.com/disk/
73
+
74
+ [cacache]: https://crates.io/crates/cacache
75
+ [cloudflare_kv]: https://developers.cloudflare.com/kv/
76
+ [dashmap]: https://github.com/xacrimon/dashmap
77
+ [etcd]: https://etcd.io/
78
+ [foundationdb]: https://www.foundationdb.org/
79
+ [persy]: https://crates.io/crates/persy
80
+ [redis]: https://redis.io/
81
+ [rocksdb]: http://rocksdb.org/
82
+ [sled]: https://crates.io/crates/sled
83
+ [redb]: https://crates.io/crates/redb
84
+ [tikv]: https://tikv.org/
85
+
86
+ [d1]: https://developers.cloudflare.com/d1/
87
+ [mongodb]: https://www.mongodb.com/
88
+ [mysql]: https://www.mysql.com/
89
+ [postgresql]: https://www.postgresql.org/
90
+ [sqlite]: https://www.sqlite.org/
91
+ [surrealdb]: https://surrealdb.com/
92
+
93
+ [ghac]: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
94
+ [memcached]: https://memcached.org/
95
+ [mini_moka]: https://github.com/moka-rs/mini-moka
96
+ [moka]: https://github.com/moka-rs/moka
97
+ [vercel_artifacts]: https://vercel.com/docs/concepts/monorepos/remote-caching
98
+
99
+ [huggingface]: https://huggingface.co/
100
+
101
+ ## Layers
102
+
103
+ OpenDAL supports the following storage [layers](https://docs.rs/opendal/latest/opendal/layers/index.html) to extend the behavior:
104
+
105
+ | Name | Depends | Description |
106
+ |---------------------------|--------------------------|---------------------------------------------------------------------------------------|
107
+ | [`AsyncBacktraceLayer`] | [async-backtrace] | Add Efficient, logical 'stack' traces of async functions for the underlying services. |
108
+ | [`AwaitTreeLayer`] | [await-tree] | Add a Instrument await-tree for actor-based applications to the underlying services. |
109
+ | [`BlockingLayer`] | [tokio] | Add blocking API support for non-blocking services. |
110
+ | [`ChaosLayer`] | [rand] | Inject chaos into underlying services for robustness test. |
111
+ | [`ConcurrentLimitLayer`] | [tokio] | Add concurrent request limit. |
112
+ | [`DtraceLayer`] | [probe] | Support User Statically-Defined Tracing(aka USDT) on Linux |
113
+ | [`LoggingLayer`] | [log] | Add log for every operations. |
114
+ | [`MetricsLayer`] | [metrics] | Add metrics for every operations. |
115
+ | [`MimeGuessLayer`] | [mime_guess] | Add `Content-Type` automatically based on the file extension in the operation path. |
116
+ | [`FastraceLayer`] | [fastrace] | Add fastrace for every operations. |
117
+ | [`OtelMetricsLayer`] | [opentelemetry::metrics] | Add opentelemetry::metrics for every operations. |
118
+ | [`OtelTraceLayer`] | [opentelemetry::trace] | Add opentelemetry::trace for every operations. |
119
+ | [`PrometheusClientLayer`] | [prometheus_client] | Add prometheus metrics for every operations. |
120
+ | [`PrometheusLayer`] | [prometheus] | Add prometheus metrics for every operations. |
121
+ | [`RetryLayer`] | [backon] | Add retry for temporary failed operations. |
122
+ | [`ThrottleLayer`] | [governor] | Add a bandwidth rate limiter to the underlying services. |
123
+ | [`TimeoutLayer`] | [tokio] | Add timeout for every operations to avoid slow or unexpected hang operations. |
124
+ | [`TracingLayer`] | [tracing] | Add tracing for every operations. |
125
+
126
+ [`AsyncBacktraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.AsyncBacktraceLayer.html
127
+ [async-backtrace]: https://github.com/tokio-rs/async-backtrace
128
+ [`AwaitTreeLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.AwaitTreeLayer.html
129
+ [await-tree]: https://github.com/risingwavelabs/await-tree
130
+ [`BlockingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.BlockingLayer.html
131
+ [tokio]: https://github.com/tokio-rs/tokio
132
+ [`ChaosLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ChaosLayer.html
133
+ [rand]: https://github.com/rust-random/rand
134
+ [`ConcurrentLimitLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ConcurrentLimitLayer.html
135
+ [`DtraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.DtraceLayer.html
136
+ [probe]: https://github.com/cuviper/probe-rs
137
+ [`LoggingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.LoggingLayer.html
138
+ [log]: https://github.com/rust-lang/log
139
+ [`MetricsLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.MetricsLayer.html
140
+ [metrics]: https://github.com/metrics-rs/metrics
141
+ [`MimeGuessLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.MimeGuessLayer.html
142
+ [mime_guess]: https://github.com/abonander/mime_guess
143
+ [`FastraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.FastraceLayer.html
144
+ [fastrace]: https://github.com/fastracelabs/fastrace
145
+ [`OtelMetricsLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.OtelMetricsLayer.html
146
+ [`OtelTraceLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.OtelTraceLayer.html
147
+ [opentelemetry::trace]: https://docs.rs/opentelemetry/latest/opentelemetry/trace/index.html
148
+ [`PrometheusClientLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.PrometheusClientLayer.html
149
+ [prometheus_client]: https://github.com/prometheus/client_rust
150
+ [`PrometheusLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.PrometheusLayer.html
151
+ [prometheus]: https://github.com/tikv/rust-prometheus
152
+ [`RetryLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.RetryLayer.html
153
+ [backon]: https://github.com/Xuanwo/backon
154
+ [`ThrottleLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.ThrottleLayer.html
155
+ [governor]: https://github.com/boinkor-net/governor
156
+ [`TimeoutLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.TimeoutLayer.html
157
+ [`TracingLayer`]: https://docs.rs/opendal/latest/opendal/layers/struct.TracingLayer.html
158
+ [tracing]: https://github.com/tokio-rs/tracing
159
+
160
+ ## Quickstart
161
+
162
+ ```rust
163
+ use opendal::Result;
164
+ use opendal::layers::LoggingLayer;
165
+ use opendal::services;
166
+ use opendal::Operator;
167
+
168
+ #[tokio::main]
169
+ async fn main() -> Result<()> {
170
+ // Pick a builder and configure it.
171
+ let mut builder = services::S3::default();
172
+ builder.bucket("test");
173
+
174
+ // Init an operator
175
+ let op = Operator::new(builder)?
176
+ // Init with logging layer enabled.
177
+ .layer(LoggingLayer::default())
178
+ .finish();
179
+
180
+ // Write data
181
+ op.write("hello.txt", "Hello, World!").await?;
182
+
183
+ // Read data
184
+ let bs = op.read("hello.txt").await?;
185
+
186
+ // Fetch metadata
187
+ let meta = op.stat("hello.txt").await?;
188
+ let mode = meta.mode();
189
+ let length = meta.content_length();
190
+
191
+ // Delete
192
+ op.delete("hello.txt").await?;
193
+
194
+ Ok(())
195
+ }
196
+ ```
197
+
198
+ ## Examples
199
+
200
+ | Name | Description |
201
+ |---------------------|---------------------------------------------------------------|
202
+ | [Basic] | Show how to use opendal to operate storage service. |
203
+ | [Concurrent Upload] | Show how to perform upload concurrently to a storage service. |
204
+ | [Multipart Upload] | Show how to perform a multipart upload to a storage service. |
205
+
206
+ [Basic]: ./examples/basic
207
+ [Concurrent Upload]: ./examples/concurrent-upload
208
+ [Multipart Upload]: ./examples/multipart-upload
209
+
210
+ ## Contributing
211
+
212
+ Check out the [CONTRIBUTING](./CONTRIBUTING.md) guide for more details on getting started with contributing to this project.
213
+
214
+ ## Used by
215
+
216
+ Check out the [users](./users.md) list for more details on who is using OpenDAL.
217
+
218
+ ## Branding
219
+
220
+ The first and most prominent mentions must use the full form: **Apache OpenDALâ„¢** of the name for any individual usage (webpage, handout, slides, etc.) Depending on the context and writing style, you should use the full form of the name sufficiently often to ensure that readers clearly understand the association of both the OpenDAL project and the OpenDAL software product to the ASF as the parent organization.
221
+
222
+ For more details, see the [Apache Product Name Usage Guide](https://www.apache.org/foundation/marks/guide).
223
+
224
+ ## License and Trademarks
225
+
226
+ Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
227
+
228
+ Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
@@ -0,0 +1,18 @@
1
+ # OpenDAL Benchmarks
2
+
3
+ Running benchmark
4
+
5
+ ```shell
6
+ OPENDAL_TEST=memory cargo bench
7
+ ```
8
+
9
+ Build flamegraph:
10
+
11
+ ```shell
12
+ cargo flamegraph --bench io -- seek --bench
13
+ ```
14
+
15
+ - `--bench io` pick the `io` target.
16
+ - `-- seek --bench`: chose the benches with `seek` and `--bench` is required by `criterion`
17
+
18
+ After `flamegraph.svg` has been generated, use browser to visit it.
@@ -0,0 +1,26 @@
1
+ # Ops Benchmark Tests
2
+
3
+ Ops Benchmark Tests measure every operation's performance on the target platform.
4
+
5
+ To support benching different backends simultaneously, we use `environment value` to carry the backend config.
6
+
7
+ ## Setup
8
+
9
+ Please copy `.env.example` to `.env` and change the values on need.
10
+
11
+ Take `fs` for example, we add config on `fs` on `/tmp`.
12
+
13
+ ```dotenv
14
+ OPENDAL_FS_ROOT=/tmp
15
+ ```
16
+
17
+ Notice: The default will skip benches if the env is not set.
18
+
19
+ ## Run
20
+
21
+ Test specific backend, take s3 for example, first set the corresponding environment variables of s3, then:
22
+
23
+ ```shell
24
+ OPENDAL_TEST=s3
25
+ cargo bench ops --features tests
26
+ ```
@@ -0,0 +1,9 @@
1
+ # Types Benchmark Tests
2
+
3
+ This benchmark contains performance testing of critical data structures in opendal types, currently including performance testing of Buffer.
4
+
5
+ ## Run
6
+
7
+ ```shell
8
+ cargo bench types --features tests
9
+ ```
@@ -0,0 +1,35 @@
1
+ # OpenDAL Benchmark VS Fs
2
+
3
+ This benchmark compares the performance of OpenDAL with the performance of the `std::fs`.
4
+
5
+ ## Goal
6
+
7
+ We expect OpenDAL to match `std::fs` in speed: the throughput of OpenDAL should be within a `5%` range of `std::fs`.
8
+
9
+ ## Usage
10
+
11
+ For test: `cargo run`
12
+
13
+ ```shell
14
+ Testing vs_fs/std_fs_read
15
+ Success
16
+ Testing vs_fs/opendal_fs_read
17
+ Success
18
+ Testing vs_fs/opendal_fs_read_with_range
19
+ Success
20
+ ```
21
+
22
+ For bench: `cargo run --release -- --bench`
23
+
24
+ ```shell
25
+ read/std_fs time: [749.57 µs 762.69 µs 777.07 µs]
26
+ thrpt: [20.108 GiB/s 20.487 GiB/s 20.845 GiB/s]
27
+
28
+ read/opendal_fs time: [750.90 µs 755.39 µs 760.49 µs]
29
+ thrpt: [20.546 GiB/s 20.685 GiB/s 20.808 GiB/s]
30
+
31
+ read/opendal_fs_with_range
32
+ time: [684.02 µs 690.77 µs 697.99 µs]
33
+ thrpt: [22.386 GiB/s 22.620 GiB/s 22.843 GiB/s]
34
+
35
+ ```
@@ -0,0 +1,55 @@
1
+ # OpenDAL Benchmark VS AWS SDK S3
2
+
3
+ This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`.
4
+
5
+ ## Goal
6
+
7
+ We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `5%` range of `aws_sdk_s3`.
8
+
9
+ ## Notes
10
+
11
+ Please run bench case separately, otherwise the result will be affected by each other since we are sharing the same runtime.
12
+
13
+ ## Usage
14
+
15
+ For test: `cargo run`
16
+
17
+ ```shell
18
+ > cargo run
19
+ Testing read/opendal_s3_reader
20
+ Success
21
+ Testing read/aws_s3_sdk_into_async_read
22
+ Success
23
+ Testing read/opendal_s3_reader_with_capacity
24
+ Success
25
+ Testing read/aws_s3_sdk_into_async_read_with_capacity
26
+ Success
27
+ ```
28
+
29
+ For bench case: `cargo run --release -- opendal_s3_reader --bench`
30
+
31
+ ```shell
32
+ > cargo run --release -- opendal_s3_reader --bench
33
+ read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
34
+ thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
35
+ ```
36
+
37
+ For bench: `cargo run --release -- --bench`
38
+
39
+ ```shell
40
+ > cargo run --release -- --bench
41
+ read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
42
+ thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
43
+
44
+ read/aws_s3_sdk_into_async_read
45
+ time: [12.527 ms 12.842 ms 13.158 ms]
46
+ thrpt: [1.1875 GiB/s 1.2168 GiB/s 1.2473 GiB/s]
47
+
48
+ read/opendal_s3_reader_with_capacity
49
+ time: [9.6098 ms 9.8133 ms 10.017 ms]
50
+ thrpt: [1.5599 GiB/s 1.5922 GiB/s 1.6259 GiB/s]
51
+
52
+ read/aws_s3_sdk_into_async_read_with_capacity
53
+ time: [9.8970 ms 10.113 ms 10.329 ms]
54
+ thrpt: [1.5128 GiB/s 1.5451 GiB/s 1.5788 GiB/s]
55
+ ```
@@ -0,0 +1,3 @@
1
+ # OpenDAL Edge Tests
2
+
3
+ OpenDAL edge tests served as edge tests for the OpenDAL project. They will have pre-set data and will test the functionality of the OpenDAL project.
@@ -0,0 +1,14 @@
1
+ # File Write on Fill Disk
2
+
3
+ Reported by [The `FsBackend` only writes partial bytes and doesn't raise any errors](https://github.com/apache/opendal/issues/3052).
4
+
5
+ Setup:
6
+
7
+ ```shell
8
+ fallocate -l 512K disk.img
9
+ mkfs disk.img
10
+ mkdir ./td
11
+ sudo mount -o loop td.img ./td
12
+ chmod a+wr ./td
13
+ ```
14
+
@@ -0,0 +1,18 @@
1
+ # S3 AWS Assume Role With Web Identity
2
+
3
+ This edge test case is for AWS s3 services that authed by `Assume Role With Web Identity`.
4
+
5
+ For setup, please configure bucket and OIDC correctly, for example:
6
+
7
+ ```yaml
8
+ - uses: actions/github-script@v7
9
+ id: id-token
10
+ with:
11
+ script: return await core.getIDToken("sts.amazonaws.com")
12
+ result-encoding: string
13
+ - name: Write ID token to file
14
+ run: echo "${{ steps.id-token.outputs.result }}" > core/tests/data/web_identity_token
15
+ ```
16
+
17
+ And configure `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`.
18
+
@@ -0,0 +1,3 @@
1
+ pkg/
2
+ node_modules/
3
+ dist/
@@ -0,0 +1,42 @@
1
+ # S3 Read on WASM
2
+
3
+ This test is used to ensure opendal can run on WASM target.
4
+
5
+ ## Setup
6
+
7
+ Start the S3 server at `http://127.0.0.1:9900`
8
+
9
+ We are using the following config values, please feel free to change based on your own needs.
10
+
11
+ ```rust
12
+ let mut cfg = S3::default();
13
+ cfg.endpoint("http://127.0.0.1:9900");
14
+ cfg.access_key_id("minioadmin");
15
+ cfg.secret_access_key("minioadmin");
16
+ cfg.bucket("opendal");
17
+ cfg.region("us-east-1");
18
+ ```
19
+
20
+ ## Install
21
+
22
+ ```shell
23
+ cargo install wasm-pack
24
+ ```
25
+
26
+ ## Build
27
+
28
+ ```shell
29
+ wasm-pack build
30
+ ```
31
+
32
+ ## Test
33
+
34
+ NOTE:
35
+
36
+ - You need to have Chrome installed.
37
+ - We can't run on node.js yet.
38
+ - We can't run on headless chrome yet.
39
+
40
+ ```shell
41
+ wasm-pack test --chrome
42
+ ```
@@ -0,0 +1,15 @@
1
+ {
2
+ "moz:firefoxOptions": {
3
+ "prefs": {
4
+ "media.navigator.streams.fake": true,
5
+ "media.navigator.permission.disabled": true
6
+ },
7
+ "args": []
8
+ },
9
+ "goog:chromeOptions": {
10
+ "args": [
11
+ "--use-fake-device-for-media-stream",
12
+ "--use-fake-ui-for-media-stream"
13
+ ]
14
+ }
15
+ }
@@ -0,0 +1,23 @@
1
+ # Apache OpenDALâ„¢ Rust Core Examples
2
+
3
+ Thank you for using OpenDAL!
4
+
5
+ Those examples are designed to help you to understand how to use OpenDAL Rust Core.
6
+
7
+ ## Setup
8
+
9
+ All examples following the same setup steps:
10
+
11
+ To run this example, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
12
+
13
+ Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`.
14
+
15
+ ```dotenv
16
+ OPENDAL_FS_ROOT=/path/to/dir
17
+ ```
18
+
19
+ into
20
+
21
+ ```dotenv
22
+ OPENDAL_FS_ROOT=/tmp/
23
+ ```
@@ -0,0 +1,15 @@
1
+ # Basic
2
+
3
+ This example will show how to use opendal to operate storage service.
4
+
5
+ ## Setup
6
+
7
+ Refer [Setup](../README.md) to setup all examples.
8
+
9
+ ## Run
10
+
11
+ Use `OPENDAL_TEST` to control which service to run example:
12
+
13
+ ```shell
14
+ OPENDAL_TEST=fs cargo run
15
+ ```
@@ -0,0 +1,15 @@
1
+ # Concurrent Upload
2
+
3
+ This example will show how to perform upload concurrently to a storage service.
4
+
5
+ ## Setup
6
+
7
+ Refer [Setup](../README.md) to setup all examples.
8
+
9
+ ## Run
10
+
11
+ Use `OPENDAL_TEST` to control which service to run example:
12
+
13
+ ```shell
14
+ OPENDAL_TEST=s3 cargo run
15
+ ```
@@ -0,0 +1,15 @@
1
+ # Multipart Upload
2
+
3
+ This example will show how to perform a multipart upload to a storage service.
4
+
5
+ ## Setup
6
+
7
+ Refer [Setup](../README.md) to setup all examples.
8
+
9
+ ## Run
10
+
11
+ Use `OPENDAL_TEST` to control which service to run example:
12
+
13
+ ```shell
14
+ OPENDAL_TEST=s3 cargo run
15
+ ```
@@ -0,0 +1,5 @@
1
+ target
2
+ corpus
3
+ artifacts
4
+ coverage
5
+ tmp
@@ -0,0 +1,68 @@
1
+ # Fuzz Test for OpenDAL
2
+
3
+ fuzz test are used to test the robustness of the code.
4
+
5
+ ## Setup
6
+
7
+
8
+
9
+ To run the fuzz tests, please copy the `.env.example`, which is at project root, to `.env` and change the values on need.
10
+
11
+ Take `fs` for example, we need to change to enable fuzz test on `fs` on `/tmp`.
12
+
13
+ ```dotenv
14
+ OPENDAL_TEST=fs
15
+ OPENDAL_FS_ROOT=/path/to/dir/
16
+ ```
17
+
18
+ into
19
+
20
+ ```dotenv
21
+ OPENDAL_TEST=fs
22
+ OPENDAL_FS_ROOT=/tmp/
23
+ ```
24
+
25
+
26
+ ## Run
27
+
28
+ List all fuzz targets.
29
+
30
+ ```bash
31
+ cargo +nightly fuzz list
32
+ ```
33
+
34
+ Build fuzz targets.
35
+
36
+ ```bash
37
+ cargo +nightly fuzz build
38
+ ```
39
+
40
+ Run a fuzz target(such as `reader`).
41
+
42
+ ```bash
43
+ cargo +nightly fuzz run fuzz_reader
44
+ ```
45
+
46
+ ## Crash Reproduction
47
+
48
+ If you want to reproduce a crash, you first need to obtain the Base64 encoded code, which usually appears at the end of a crash report, and store it in a file.
49
+
50
+ Alternatively, if you already have the crash file, you can skip this step.
51
+
52
+ ```bash
53
+ echo "Base64" > .crash
54
+ ```
55
+
56
+ Print the `std::fmt::Debug` output for an input.
57
+
58
+ ```bash
59
+ cargo +nightly fuzz fmt fuzz_target .crash
60
+ ```
61
+
62
+ Rerun the fuzz test with the input.
63
+
64
+ ```bash
65
+ cargo +nightly fuzz run fuzz_target .crash
66
+ ```
67
+
68
+ For more details, please visit [cargo fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/tutorial.html) or run the command cargo fuzz --help.