opendal 0.1.6.pre.rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +20 -0
- data/.tool-versions +1 -0
- data/.yardopts +1 -0
- data/Cargo.toml +65 -0
- data/DEPENDENCIES.md +9 -0
- data/DEPENDENCIES.rust.tsv +277 -0
- data/Gemfile +35 -0
- data/README.md +159 -0
- data/Rakefile +149 -0
- data/build.rs +22 -0
- data/core/CHANGELOG.md +4929 -0
- data/core/CONTRIBUTING.md +61 -0
- data/core/Cargo.lock +10259 -0
- data/core/Cargo.toml +437 -0
- data/core/DEPENDENCIES.md +3 -0
- data/core/DEPENDENCIES.rust.tsv +185 -0
- data/core/LICENSE +201 -0
- data/core/README.md +228 -0
- data/core/benches/README.md +18 -0
- data/core/benches/ops/README.md +26 -0
- data/core/benches/ops/main.rs +25 -0
- data/core/benches/ops/read.rs +100 -0
- data/core/benches/ops/utils.rs +59 -0
- data/core/benches/ops/write.rs +106 -0
- data/core/benches/types/README.md +9 -0
- data/core/benches/types/buffer.rs +114 -0
- data/core/benches/types/main.rs +23 -0
- data/core/benches/types/tasks.rs +64 -0
- data/core/benches/vs_fs/Cargo.toml +32 -0
- data/core/benches/vs_fs/README.md +35 -0
- data/core/benches/vs_fs/src/main.rs +83 -0
- data/core/benches/vs_s3/Cargo.toml +38 -0
- data/core/benches/vs_s3/README.md +55 -0
- data/core/benches/vs_s3/src/main.rs +123 -0
- data/core/edge/README.md +3 -0
- data/core/edge/file_write_on_full_disk/Cargo.toml +31 -0
- data/core/edge/file_write_on_full_disk/README.md +14 -0
- data/core/edge/file_write_on_full_disk/src/main.rs +43 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml +30 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs +34 -0
- data/core/edge/s3_read_on_wasm/.gitignore +3 -0
- data/core/edge/s3_read_on_wasm/Cargo.toml +38 -0
- data/core/edge/s3_read_on_wasm/README.md +42 -0
- data/core/edge/s3_read_on_wasm/src/lib.rs +60 -0
- data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
- data/core/examples/README.md +23 -0
- data/core/examples/basic/Cargo.toml +29 -0
- data/core/examples/basic/README.md +15 -0
- data/core/examples/basic/src/main.rs +51 -0
- data/core/examples/concurrent-upload/Cargo.toml +29 -0
- data/core/examples/concurrent-upload/README.md +15 -0
- data/core/examples/concurrent-upload/src/main.rs +68 -0
- data/core/examples/multipart-upload/Cargo.toml +29 -0
- data/core/examples/multipart-upload/README.md +15 -0
- data/core/examples/multipart-upload/src/main.rs +56 -0
- data/core/fuzz/.gitignore +5 -0
- data/core/fuzz/Cargo.toml +92 -0
- data/core/fuzz/README.md +68 -0
- data/core/fuzz/fuzz_reader.rs +102 -0
- data/core/fuzz/fuzz_writer.rs +123 -0
- data/core/src/blocking/delete.rs +74 -0
- data/core/src/blocking/list.rs +71 -0
- data/core/src/blocking/mod.rs +33 -0
- data/core/src/blocking/operator.rs +729 -0
- data/core/src/blocking/read/buffer_iterator.rs +66 -0
- data/core/src/blocking/read/mod.rs +27 -0
- data/core/src/blocking/read/reader.rs +124 -0
- data/core/src/blocking/read/std_bytes_iterator.rs +69 -0
- data/core/src/blocking/read/std_reader.rs +95 -0
- data/core/src/blocking/write/mod.rs +22 -0
- data/core/src/blocking/write/std_writer.rs +82 -0
- data/core/src/blocking/write/writer.rs +109 -0
- data/core/src/docs/comparisons/mod.rs +30 -0
- data/core/src/docs/comparisons/vs_object_store.md +183 -0
- data/core/src/docs/concepts.rs +135 -0
- data/core/src/docs/internals/accessor.rs +306 -0
- data/core/src/docs/internals/layer.rs +42 -0
- data/core/src/docs/internals/mod.rs +62 -0
- data/core/src/docs/mod.rs +43 -0
- data/core/src/docs/performance/concurrent_write.md +101 -0
- data/core/src/docs/performance/http_optimization.md +124 -0
- data/core/src/docs/performance/mod.rs +32 -0
- data/core/src/docs/rfcs/0000_example.md +74 -0
- data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
- data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
- data/core/src/docs/rfcs/0044_error_handle.md +198 -0
- data/core/src/docs/rfcs/0057_auto_region.md +160 -0
- data/core/src/docs/rfcs/0069_object_stream.md +145 -0
- data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
- data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
- data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
- data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
- data/core/src/docs/rfcs/0221_create_dir.md +89 -0
- data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
- data/core/src/docs/rfcs/0293_object_id.md +67 -0
- data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
- data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
- data/core/src/docs/rfcs/0413_presign.md +154 -0
- data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
- data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
- data/core/src/docs/rfcs/0438_multipart.md +163 -0
- data/core/src/docs/rfcs/0443_gateway.md +73 -0
- data/core/src/docs/rfcs/0501_new_builder.md +111 -0
- data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
- data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
- data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
- data/core/src/docs/rfcs/0623_redis_service.md +300 -0
- data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
- data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
- data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
- data/core/src/docs/rfcs/0926_object_reader.md +93 -0
- data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
- data/core/src/docs/rfcs/1085_object_handler.md +73 -0
- data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
- data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
- data/core/src/docs/rfcs/1420_object_writer.md +147 -0
- data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
- data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
- data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
- data/core/src/docs/rfcs/2133_append_api.md +88 -0
- data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
- data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
- data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
- data/core/src/docs/rfcs/2774_lister_api.md +66 -0
- data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
- data/core/src/docs/rfcs/2852_native_capability.md +58 -0
- data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
- data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
- data/core/src/docs/rfcs/3197_config.md +237 -0
- data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
- data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
- data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
- data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
- data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
- data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
- data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
- data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
- data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
- data/core/src/docs/rfcs/4638_executor.md +215 -0
- data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
- data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
- data/core/src/docs/rfcs/5479_context.md +140 -0
- data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
- data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
- data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
- data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
- data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
- data/core/src/docs/rfcs/6209_glob_support.md +132 -0
- data/core/src/docs/rfcs/6213_options_api.md +142 -0
- data/core/src/docs/rfcs/README.md +62 -0
- data/core/src/docs/rfcs/mod.rs +278 -0
- data/core/src/docs/upgrade.md +1556 -0
- data/core/src/layers/async_backtrace.rs +174 -0
- data/core/src/layers/await_tree.rs +202 -0
- data/core/src/layers/capability_check.rs +239 -0
- data/core/src/layers/chaos.rs +170 -0
- data/core/src/layers/complete.rs +385 -0
- data/core/src/layers/concurrent_limit.rs +322 -0
- data/core/src/layers/correctness_check.rs +440 -0
- data/core/src/layers/dtrace.rs +294 -0
- data/core/src/layers/error_context.rs +310 -0
- data/core/src/layers/fastmetrics.rs +525 -0
- data/core/src/layers/fastrace.rs +271 -0
- data/core/src/layers/http_client.rs +206 -0
- data/core/src/layers/immutable_index.rs +408 -0
- data/core/src/layers/logging.rs +842 -0
- data/core/src/layers/metrics.rs +182 -0
- data/core/src/layers/mime_guess.rs +199 -0
- data/core/src/layers/mod.rs +130 -0
- data/core/src/layers/observe/metrics.rs +936 -0
- data/core/src/layers/observe/mod.rs +93 -0
- data/core/src/layers/otelmetrics.rs +496 -0
- data/core/src/layers/oteltrace.rs +203 -0
- data/core/src/layers/prometheus.rs +686 -0
- data/core/src/layers/prometheus_client.rs +519 -0
- data/core/src/layers/retry.rs +933 -0
- data/core/src/layers/throttle.rs +204 -0
- data/core/src/layers/timeout.rs +513 -0
- data/core/src/layers/tracing.rs +349 -0
- data/core/src/layers/type_eraser.rs +91 -0
- data/core/src/lib.rs +204 -0
- data/core/src/raw/accessor.rs +856 -0
- data/core/src/raw/adapters/kv/api.rs +164 -0
- data/core/src/raw/adapters/kv/backend.rs +253 -0
- data/core/src/raw/adapters/kv/mod.rs +31 -0
- data/core/src/raw/adapters/mod.rs +50 -0
- data/core/src/raw/adapters/typed_kv/api.rs +171 -0
- data/core/src/raw/adapters/typed_kv/backend.rs +279 -0
- data/core/src/raw/adapters/typed_kv/mod.rs +29 -0
- data/core/src/raw/atomic_util.rs +57 -0
- data/core/src/raw/azure.rs +570 -0
- data/core/src/raw/chrono_util.rs +109 -0
- data/core/src/raw/enum_utils.rs +201 -0
- data/core/src/raw/futures_util.rs +470 -0
- data/core/src/raw/http_util/body.rs +144 -0
- data/core/src/raw/http_util/bytes_content_range.rs +239 -0
- data/core/src/raw/http_util/bytes_range.rs +260 -0
- data/core/src/raw/http_util/client.rs +276 -0
- data/core/src/raw/http_util/error.rs +68 -0
- data/core/src/raw/http_util/header.rs +356 -0
- data/core/src/raw/http_util/mod.rs +78 -0
- data/core/src/raw/http_util/multipart.rs +1180 -0
- data/core/src/raw/http_util/uri.rs +190 -0
- data/core/src/raw/layer.rs +295 -0
- data/core/src/raw/mod.rs +101 -0
- data/core/src/raw/oio/buf/flex_buf.rs +118 -0
- data/core/src/raw/oio/buf/mod.rs +25 -0
- data/core/src/raw/oio/buf/pooled_buf.rs +126 -0
- data/core/src/raw/oio/buf/queue_buf.rs +117 -0
- data/core/src/raw/oio/delete/api.rs +102 -0
- data/core/src/raw/oio/delete/batch_delete.rs +127 -0
- data/core/src/raw/oio/delete/mod.rs +30 -0
- data/core/src/raw/oio/delete/one_shot_delete.rs +79 -0
- data/core/src/raw/oio/entry.rs +89 -0
- data/core/src/raw/oio/list/api.rs +69 -0
- data/core/src/raw/oio/list/flat_list.rs +137 -0
- data/core/src/raw/oio/list/hierarchy_list.rs +135 -0
- data/core/src/raw/oio/list/mod.rs +35 -0
- data/core/src/raw/oio/list/page_list.rs +105 -0
- data/core/src/raw/oio/list/prefix_list.rs +64 -0
- data/core/src/raw/oio/mod.rs +40 -0
- data/core/src/raw/oio/read/api.rs +119 -0
- data/core/src/raw/oio/read/mod.rs +21 -0
- data/core/src/raw/oio/write/api.rs +103 -0
- data/core/src/raw/oio/write/append_write.rs +111 -0
- data/core/src/raw/oio/write/block_write.rs +405 -0
- data/core/src/raw/oio/write/mod.rs +42 -0
- data/core/src/raw/oio/write/multipart_write.rs +518 -0
- data/core/src/raw/oio/write/one_shot_write.rs +77 -0
- data/core/src/raw/oio/write/position_write.rs +284 -0
- data/core/src/raw/operation.rs +88 -0
- data/core/src/raw/ops.rs +917 -0
- data/core/src/raw/path.rs +451 -0
- data/core/src/raw/path_cache.rs +244 -0
- data/core/src/raw/rps.rs +249 -0
- data/core/src/raw/serde_util.rs +423 -0
- data/core/src/raw/std_io_util.rs +65 -0
- data/core/src/raw/tests/mod.rs +30 -0
- data/core/src/raw/tests/read.rs +116 -0
- data/core/src/raw/tests/utils.rs +80 -0
- data/core/src/raw/tests/write.rs +79 -0
- data/core/src/raw/tokio_util.rs +24 -0
- data/core/src/raw/version.rs +19 -0
- data/core/src/services/aliyun_drive/backend.rs +421 -0
- data/core/src/services/aliyun_drive/config.rs +72 -0
- data/core/src/services/aliyun_drive/core.rs +651 -0
- data/core/src/services/aliyun_drive/delete.rs +51 -0
- data/core/src/services/aliyun_drive/docs.md +61 -0
- data/core/src/services/aliyun_drive/error.rs +56 -0
- data/core/src/services/aliyun_drive/lister.rs +134 -0
- data/core/src/services/aliyun_drive/mod.rs +39 -0
- data/core/src/services/aliyun_drive/writer.rs +114 -0
- data/core/src/services/alluxio/backend.rs +257 -0
- data/core/src/services/alluxio/config.rs +50 -0
- data/core/src/services/alluxio/core.rs +367 -0
- data/core/src/services/alluxio/delete.rs +38 -0
- data/core/src/services/alluxio/docs.md +45 -0
- data/core/src/services/alluxio/error.rs +99 -0
- data/core/src/services/alluxio/lister.rs +73 -0
- data/core/src/services/alluxio/mod.rs +39 -0
- data/core/src/services/alluxio/writer.rs +74 -0
- data/core/src/services/azblob/backend.rs +594 -0
- data/core/src/services/azblob/config.rs +220 -0
- data/core/src/services/azblob/core.rs +937 -0
- data/core/src/services/azblob/delete.rs +108 -0
- data/core/src/services/azblob/docs.md +77 -0
- data/core/src/services/azblob/error.rs +164 -0
- data/core/src/services/azblob/lister.rs +107 -0
- data/core/src/services/azblob/mod.rs +38 -0
- data/core/src/services/azblob/writer.rs +177 -0
- data/core/src/services/azdls/backend.rs +435 -0
- data/core/src/services/azdls/config.rs +89 -0
- data/core/src/services/azdls/core.rs +388 -0
- data/core/src/services/azdls/delete.rs +48 -0
- data/core/src/services/azdls/docs.md +73 -0
- data/core/src/services/azdls/error.rs +107 -0
- data/core/src/services/azdls/lister.rs +165 -0
- data/core/src/services/azdls/mod.rs +38 -0
- data/core/src/services/azdls/writer.rs +129 -0
- data/core/src/services/azfile/backend.rs +373 -0
- data/core/src/services/azfile/config.rs +61 -0
- data/core/src/services/azfile/core.rs +435 -0
- data/core/src/services/azfile/delete.rs +51 -0
- data/core/src/services/azfile/docs.md +65 -0
- data/core/src/services/azfile/error.rs +108 -0
- data/core/src/services/azfile/lister.rs +217 -0
- data/core/src/services/azfile/mod.rs +39 -0
- data/core/src/services/azfile/writer.rs +92 -0
- data/core/src/services/b2/backend.rs +434 -0
- data/core/src/services/b2/config.rs +64 -0
- data/core/src/services/b2/core.rs +742 -0
- data/core/src/services/b2/delete.rs +56 -0
- data/core/src/services/b2/docs.md +54 -0
- data/core/src/services/b2/error.rs +132 -0
- data/core/src/services/b2/lister.rs +110 -0
- data/core/src/services/b2/mod.rs +39 -0
- data/core/src/services/b2/writer.rs +189 -0
- data/core/src/services/cacache/backend.rs +160 -0
- data/core/src/services/cacache/config.rs +28 -0
- data/core/src/services/cacache/core.rs +96 -0
- data/core/src/services/cacache/delete.rs +39 -0
- data/core/src/services/cacache/docs.md +38 -0
- data/core/src/services/cacache/mod.rs +34 -0
- data/core/src/services/cacache/writer.rs +61 -0
- data/core/src/services/cloudflare_kv/backend.rs +513 -0
- data/core/src/services/cloudflare_kv/config.rs +55 -0
- data/core/src/services/cloudflare_kv/core.rs +168 -0
- data/core/src/services/cloudflare_kv/delete.rs +119 -0
- data/core/src/services/cloudflare_kv/docs.md +21 -0
- data/core/src/services/cloudflare_kv/error.rs +79 -0
- data/core/src/services/cloudflare_kv/lister.rs +170 -0
- data/core/src/services/cloudflare_kv/mod.rs +39 -0
- data/core/src/services/cloudflare_kv/model.rs +76 -0
- data/core/src/services/cloudflare_kv/writer.rs +68 -0
- data/core/src/services/compfs/backend.rs +290 -0
- data/core/src/services/compfs/config.rs +30 -0
- data/core/src/services/compfs/core.rs +159 -0
- data/core/src/services/compfs/delete.rs +53 -0
- data/core/src/services/compfs/lister.rs +98 -0
- data/core/src/services/compfs/mod.rs +38 -0
- data/core/src/services/compfs/reader.rs +79 -0
- data/core/src/services/compfs/writer.rs +90 -0
- data/core/src/services/cos/backend.rs +442 -0
- data/core/src/services/cos/config.rs +54 -0
- data/core/src/services/cos/core.rs +761 -0
- data/core/src/services/cos/delete.rs +48 -0
- data/core/src/services/cos/docs.md +55 -0
- data/core/src/services/cos/error.rs +105 -0
- data/core/src/services/cos/lister.rs +237 -0
- data/core/src/services/cos/mod.rs +39 -0
- data/core/src/services/cos/writer.rs +234 -0
- data/core/src/services/d1/backend.rs +330 -0
- data/core/src/services/d1/config.rs +55 -0
- data/core/src/services/d1/docs.md +48 -0
- data/core/src/services/d1/error.rs +79 -0
- data/core/src/services/d1/mod.rs +29 -0
- data/core/src/services/d1/model.rs +125 -0
- data/core/src/services/dashmap/backend.rs +203 -0
- data/core/src/services/dashmap/config.rs +37 -0
- data/core/src/services/dashmap/core.rs +61 -0
- data/core/src/services/dashmap/delete.rs +40 -0
- data/core/src/services/dashmap/docs.md +38 -0
- data/core/src/services/dashmap/lister.rs +63 -0
- data/core/src/services/dashmap/mod.rs +36 -0
- data/core/src/services/dashmap/writer.rs +87 -0
- data/core/src/services/dbfs/backend.rs +258 -0
- data/core/src/services/dbfs/config.rs +48 -0
- data/core/src/services/dbfs/core.rs +191 -0
- data/core/src/services/dbfs/delete.rs +49 -0
- data/core/src/services/dbfs/docs.md +57 -0
- data/core/src/services/dbfs/error.rs +74 -0
- data/core/src/services/dbfs/lister.rs +96 -0
- data/core/src/services/dbfs/mod.rs +38 -0
- data/core/src/services/dbfs/writer.rs +64 -0
- data/core/src/services/dropbox/backend.rs +187 -0
- data/core/src/services/dropbox/builder.rs +222 -0
- data/core/src/services/dropbox/config.rs +47 -0
- data/core/src/services/dropbox/core.rs +496 -0
- data/core/src/services/dropbox/delete.rs +54 -0
- data/core/src/services/dropbox/docs.md +64 -0
- data/core/src/services/dropbox/error.rs +85 -0
- data/core/src/services/dropbox/lister.rs +117 -0
- data/core/src/services/dropbox/mod.rs +40 -0
- data/core/src/services/dropbox/writer.rs +51 -0
- data/core/src/services/etcd/backend.rs +345 -0
- data/core/src/services/etcd/config.rs +86 -0
- data/core/src/services/etcd/core.rs +143 -0
- data/core/src/services/etcd/deleter.rs +41 -0
- data/core/src/services/etcd/docs.md +45 -0
- data/core/src/services/etcd/error.rs +26 -0
- data/core/src/services/etcd/lister.rs +79 -0
- data/core/src/services/etcd/mod.rs +36 -0
- data/core/src/services/etcd/writer.rs +61 -0
- data/core/src/services/foundationdb/backend.rs +171 -0
- data/core/src/services/foundationdb/config.rs +45 -0
- data/core/src/services/foundationdb/docs.md +42 -0
- data/core/src/services/foundationdb/mod.rs +24 -0
- data/core/src/services/fs/backend.rs +299 -0
- data/core/src/services/fs/config.rs +33 -0
- data/core/src/services/fs/core.rs +227 -0
- data/core/src/services/fs/delete.rs +53 -0
- data/core/src/services/fs/docs.md +49 -0
- data/core/src/services/fs/error.rs +31 -0
- data/core/src/services/fs/lister.rs +81 -0
- data/core/src/services/fs/mod.rs +40 -0
- data/core/src/services/fs/reader.rs +83 -0
- data/core/src/services/fs/writer.rs +212 -0
- data/core/src/services/ftp/backend.rs +388 -0
- data/core/src/services/ftp/config.rs +46 -0
- data/core/src/services/ftp/core.rs +136 -0
- data/core/src/services/ftp/delete.rs +62 -0
- data/core/src/services/ftp/docs.md +42 -0
- data/core/src/services/ftp/err.rs +47 -0
- data/core/src/services/ftp/lister.rs +72 -0
- data/core/src/services/ftp/mod.rs +41 -0
- data/core/src/services/ftp/reader.rs +84 -0
- data/core/src/services/ftp/writer.rs +122 -0
- data/core/src/services/gcs/backend.rs +499 -0
- data/core/src/services/gcs/config.rs +168 -0
- data/core/src/services/gcs/core.rs +1079 -0
- data/core/src/services/gcs/delete.rs +98 -0
- data/core/src/services/gcs/docs.md +76 -0
- data/core/src/services/gcs/error.rs +122 -0
- data/core/src/services/gcs/lister.rs +136 -0
- data/core/src/services/gcs/mod.rs +40 -0
- data/core/src/services/gcs/uri.rs +75 -0
- data/core/src/services/gcs/writer.rs +163 -0
- data/core/src/services/gdrive/backend.rs +176 -0
- data/core/src/services/gdrive/builder.rs +228 -0
- data/core/src/services/gdrive/config.rs +47 -0
- data/core/src/services/gdrive/core.rs +499 -0
- data/core/src/services/gdrive/delete.rs +57 -0
- data/core/src/services/gdrive/docs.md +65 -0
- data/core/src/services/gdrive/error.rs +80 -0
- data/core/src/services/gdrive/lister.rs +110 -0
- data/core/src/services/gdrive/mod.rs +40 -0
- data/core/src/services/gdrive/writer.rs +77 -0
- data/core/src/services/ghac/backend.rs +285 -0
- data/core/src/services/ghac/config.rs +36 -0
- data/core/src/services/ghac/core.rs +459 -0
- data/core/src/services/ghac/docs.md +84 -0
- data/core/src/services/ghac/error.rs +52 -0
- data/core/src/services/ghac/mod.rs +35 -0
- data/core/src/services/ghac/writer.rs +201 -0
- data/core/src/services/github/backend.rs +285 -0
- data/core/src/services/github/config.rs +59 -0
- data/core/src/services/github/core.rs +351 -0
- data/core/src/services/github/delete.rs +41 -0
- data/core/src/services/github/docs.md +52 -0
- data/core/src/services/github/error.rs +101 -0
- data/core/src/services/github/lister.rs +112 -0
- data/core/src/services/github/mod.rs +38 -0
- data/core/src/services/github/writer.rs +51 -0
- data/core/src/services/gridfs/backend.rs +166 -0
- data/core/src/services/gridfs/config.rs +50 -0
- data/core/src/services/gridfs/core.rs +154 -0
- data/core/src/services/gridfs/docs.md +46 -0
- data/core/src/services/gridfs/mod.rs +26 -0
- data/core/src/services/hdfs/backend.rs +413 -0
- data/core/src/services/hdfs/config.rs +59 -0
- data/core/src/services/hdfs/delete.rs +62 -0
- data/core/src/services/hdfs/docs.md +140 -0
- data/core/src/services/hdfs/lister.rs +70 -0
- data/core/src/services/hdfs/mod.rs +36 -0
- data/core/src/services/hdfs/reader.rs +79 -0
- data/core/src/services/hdfs/writer.rs +104 -0
- data/core/src/services/hdfs_native/backend.rs +340 -0
- data/core/src/services/hdfs_native/config.rs +45 -0
- data/core/src/services/hdfs_native/delete.rs +47 -0
- data/core/src/services/hdfs_native/docs.md +35 -0
- data/core/src/services/hdfs_native/error.rs +59 -0
- data/core/src/services/hdfs_native/lister.rs +85 -0
- data/core/src/services/hdfs_native/mod.rs +39 -0
- data/core/src/services/hdfs_native/reader.rs +62 -0
- data/core/src/services/hdfs_native/writer.rs +61 -0
- data/core/src/services/http/backend.rs +291 -0
- data/core/src/services/http/config.rs +49 -0
- data/core/src/services/http/core.rs +125 -0
- data/core/src/services/http/docs.md +45 -0
- data/core/src/services/http/error.rs +53 -0
- data/core/src/services/http/mod.rs +32 -0
- data/core/src/services/huggingface/backend.rs +289 -0
- data/core/src/services/huggingface/config.rs +75 -0
- data/core/src/services/huggingface/core.rs +406 -0
- data/core/src/services/huggingface/docs.md +61 -0
- data/core/src/services/huggingface/error.rs +93 -0
- data/core/src/services/huggingface/lister.rs +91 -0
- data/core/src/services/huggingface/mod.rs +34 -0
- data/core/src/services/ipfs/backend.rs +257 -0
- data/core/src/services/ipfs/config.rs +32 -0
- data/core/src/services/ipfs/core.rs +239 -0
- data/core/src/services/ipfs/docs.md +45 -0
- data/core/src/services/ipfs/error.rs +52 -0
- data/core/src/services/ipfs/ipld.rs +162 -0
- data/core/src/services/ipfs/mod.rs +34 -0
- data/core/src/services/ipmfs/backend.rs +147 -0
- data/core/src/services/ipmfs/builder.rs +166 -0
- data/core/src/services/ipmfs/config.rs +32 -0
- data/core/src/services/ipmfs/core.rs +142 -0
- data/core/src/services/ipmfs/delete.rs +48 -0
- data/core/src/services/ipmfs/docs.md +14 -0
- data/core/src/services/ipmfs/error.rs +83 -0
- data/core/src/services/ipmfs/lister.rs +135 -0
- data/core/src/services/ipmfs/mod.rs +40 -0
- data/core/src/services/ipmfs/writer.rs +49 -0
- data/core/src/services/koofr/backend.rs +361 -0
- data/core/src/services/koofr/config.rs +50 -0
- data/core/src/services/koofr/core.rs +458 -0
- data/core/src/services/koofr/delete.rs +50 -0
- data/core/src/services/koofr/docs.md +51 -0
- data/core/src/services/koofr/error.rs +72 -0
- data/core/src/services/koofr/lister.rs +88 -0
- data/core/src/services/koofr/mod.rs +38 -0
- data/core/src/services/koofr/writer.rs +53 -0
- data/core/src/services/lakefs/backend.rs +309 -0
- data/core/src/services/lakefs/config.rs +81 -0
- data/core/src/services/lakefs/core.rs +261 -0
- data/core/src/services/lakefs/delete.rs +54 -0
- data/core/src/services/lakefs/docs.md +62 -0
- data/core/src/services/lakefs/error.rs +93 -0
- data/core/src/services/lakefs/lister.rs +120 -0
- data/core/src/services/lakefs/mod.rs +38 -0
- data/core/src/services/lakefs/writer.rs +50 -0
- data/core/src/services/memcached/backend.rs +284 -0
- data/core/src/services/memcached/binary.rs +289 -0
- data/core/src/services/memcached/config.rs +43 -0
- data/core/src/services/memcached/docs.md +47 -0
- data/core/src/services/memcached/mod.rs +27 -0
- data/core/src/services/memory/backend.rs +205 -0
- data/core/src/services/memory/config.rs +30 -0
- data/core/src/services/memory/core.rs +80 -0
- data/core/src/services/memory/delete.rs +42 -0
- data/core/src/services/memory/docs.md +36 -0
- data/core/src/services/memory/lister.rs +56 -0
- data/core/src/services/memory/mod.rs +36 -0
- data/core/src/services/memory/writer.rs +85 -0
- data/core/src/services/mini_moka/backend.rs +260 -0
- data/core/src/services/mini_moka/config.rs +56 -0
- data/core/src/services/mini_moka/core.rs +52 -0
- data/core/src/services/mini_moka/delete.rs +42 -0
- data/core/src/services/mini_moka/docs.md +19 -0
- data/core/src/services/mini_moka/lister.rs +68 -0
- data/core/src/services/mini_moka/mod.rs +36 -0
- data/core/src/services/mini_moka/writer.rs +84 -0
- data/core/src/services/mod.rs +206 -0
- data/core/src/services/moka/backend.rs +326 -0
- data/core/src/services/moka/config.rs +59 -0
- data/core/src/services/moka/core.rs +62 -0
- data/core/src/services/moka/delete.rs +42 -0
- data/core/src/services/moka/docs.md +42 -0
- data/core/src/services/moka/lister.rs +65 -0
- data/core/src/services/moka/mod.rs +41 -0
- data/core/src/services/moka/writer.rs +83 -0
- data/core/src/services/mongodb/backend.rs +291 -0
- data/core/src/services/mongodb/config.rs +54 -0
- data/core/src/services/mongodb/docs.md +49 -0
- data/core/src/services/mongodb/mod.rs +24 -0
- data/core/src/services/monoiofs/backend.rs +238 -0
- data/core/src/services/monoiofs/config.rs +34 -0
- data/core/src/services/monoiofs/core.rs +313 -0
- data/core/src/services/monoiofs/delete.rs +64 -0
- data/core/src/services/monoiofs/docs.md +46 -0
- data/core/src/services/monoiofs/mod.rs +36 -0
- data/core/src/services/monoiofs/reader.rs +147 -0
- data/core/src/services/monoiofs/writer.rs +189 -0
- data/core/src/services/mysql/backend.rs +256 -0
- data/core/src/services/mysql/config.rs +66 -0
- data/core/src/services/mysql/docs.md +47 -0
- data/core/src/services/mysql/mod.rs +24 -0
- data/core/src/services/obs/backend.rs +442 -0
- data/core/src/services/obs/config.rs +53 -0
- data/core/src/services/obs/core.rs +608 -0
- data/core/src/services/obs/delete.rs +48 -0
- data/core/src/services/obs/docs.md +54 -0
- data/core/src/services/obs/error.rs +106 -0
- data/core/src/services/obs/lister.rs +101 -0
- data/core/src/services/obs/mod.rs +38 -0
- data/core/src/services/obs/writer.rs +235 -0
- data/core/src/services/onedrive/backend.rs +127 -0
- data/core/src/services/onedrive/builder.rs +236 -0
- data/core/src/services/onedrive/config.rs +49 -0
- data/core/src/services/onedrive/core.rs +691 -0
- data/core/src/services/onedrive/delete.rs +47 -0
- data/core/src/services/onedrive/docs.md +115 -0
- data/core/src/services/onedrive/error.rs +61 -0
- data/core/src/services/onedrive/graph_model.rs +425 -0
- data/core/src/services/onedrive/lister.rs +150 -0
- data/core/src/services/onedrive/mod.rs +42 -0
- data/core/src/services/onedrive/writer.rs +168 -0
- data/core/src/services/opfs/backend.rs +50 -0
- data/core/src/services/opfs/config.rs +25 -0
- data/core/src/services/opfs/core.rs +74 -0
- data/core/src/services/opfs/docs.md +18 -0
- data/core/src/services/opfs/error.rs +27 -0
- data/core/src/services/opfs/mod.rs +30 -0
- data/core/src/services/opfs/utils.rs +70 -0
- data/core/src/services/oss/backend.rs +734 -0
- data/core/src/services/oss/config.rs +113 -0
- data/core/src/services/oss/core.rs +1088 -0
- data/core/src/services/oss/delete.rs +109 -0
- data/core/src/services/oss/docs.md +74 -0
- data/core/src/services/oss/error.rs +109 -0
- data/core/src/services/oss/lister.rs +256 -0
- data/core/src/services/oss/mod.rs +38 -0
- data/core/src/services/oss/writer.rs +228 -0
- data/core/src/services/pcloud/backend.rs +358 -0
- data/core/src/services/pcloud/config.rs +51 -0
- data/core/src/services/pcloud/core.rs +461 -0
- data/core/src/services/pcloud/delete.rs +66 -0
- data/core/src/services/pcloud/docs.md +51 -0
- data/core/src/services/pcloud/error.rs +88 -0
- data/core/src/services/pcloud/lister.rs +95 -0
- data/core/src/services/pcloud/mod.rs +38 -0
- data/core/src/services/pcloud/writer.rs +66 -0
- data/core/src/services/persy/backend.rs +226 -0
- data/core/src/services/persy/config.rs +32 -0
- data/core/src/services/persy/docs.md +43 -0
- data/core/src/services/persy/mod.rs +24 -0
- data/core/src/services/postgresql/backend.rs +258 -0
- data/core/src/services/postgresql/config.rs +66 -0
- data/core/src/services/postgresql/docs.md +47 -0
- data/core/src/services/postgresql/mod.rs +24 -0
- data/core/src/services/redb/backend.rs +280 -0
- data/core/src/services/redb/config.rs +34 -0
- data/core/src/services/redb/docs.md +41 -0
- data/core/src/services/redb/mod.rs +24 -0
- data/core/src/services/redis/backend.rs +442 -0
- data/core/src/services/redis/config.rs +79 -0
- data/core/src/services/redis/core.rs +209 -0
- data/core/src/services/redis/delete.rs +40 -0
- data/core/src/services/redis/docs.md +43 -0
- data/core/src/services/redis/mod.rs +34 -0
- data/core/src/services/redis/writer.rs +57 -0
- data/core/src/services/rocksdb/backend.rs +159 -0
- data/core/src/services/rocksdb/config.rs +34 -0
- data/core/src/services/rocksdb/docs.md +54 -0
- data/core/src/services/rocksdb/mod.rs +24 -0
- data/core/src/services/s3/backend.rs +1293 -0
- data/core/src/services/s3/compatible_services.md +126 -0
- data/core/src/services/s3/config.rs +327 -0
- data/core/src/services/s3/core.rs +1741 -0
- data/core/src/services/s3/delete.rs +109 -0
- data/core/src/services/s3/docs.md +244 -0
- data/core/src/services/s3/error.rs +171 -0
- data/core/src/services/s3/lister.rs +405 -0
- data/core/src/services/s3/mod.rs +38 -0
- data/core/src/services/s3/writer.rs +262 -0
- data/core/src/services/seafile/backend.rs +297 -0
- data/core/src/services/seafile/config.rs +56 -0
- data/core/src/services/seafile/core.rs +475 -0
- data/core/src/services/seafile/delete.rs +40 -0
- data/core/src/services/seafile/docs.md +54 -0
- data/core/src/services/seafile/error.rs +86 -0
- data/core/src/services/seafile/lister.rs +83 -0
- data/core/src/services/seafile/mod.rs +38 -0
- data/core/src/services/seafile/writer.rs +55 -0
- data/core/src/services/sftp/backend.rs +397 -0
- data/core/src/services/sftp/config.rs +50 -0
- data/core/src/services/sftp/core.rs +154 -0
- data/core/src/services/sftp/delete.rs +55 -0
- data/core/src/services/sftp/docs.md +49 -0
- data/core/src/services/sftp/error.rs +57 -0
- data/core/src/services/sftp/lister.rs +88 -0
- data/core/src/services/sftp/mod.rs +42 -0
- data/core/src/services/sftp/reader.rs +78 -0
- data/core/src/services/sftp/utils.rs +51 -0
- data/core/src/services/sftp/writer.rs +67 -0
- data/core/src/services/sled/backend.rs +194 -0
- data/core/src/services/sled/config.rs +45 -0
- data/core/src/services/sled/docs.md +39 -0
- data/core/src/services/sled/mod.rs +24 -0
- data/core/src/services/sqlite/backend.rs +326 -0
- data/core/src/services/sqlite/config.rs +70 -0
- data/core/src/services/sqlite/docs.md +46 -0
- data/core/src/services/sqlite/mod.rs +24 -0
- data/core/src/services/surrealdb/backend.rs +365 -0
- data/core/src/services/surrealdb/config.rs +64 -0
- data/core/src/services/surrealdb/docs.md +54 -0
- data/core/src/services/surrealdb/mod.rs +24 -0
- data/core/src/services/swift/backend.rs +275 -0
- data/core/src/services/swift/compatible_services.md +53 -0
- data/core/src/services/swift/config.rs +53 -0
- data/core/src/services/swift/core.rs +310 -0
- data/core/src/services/swift/delete.rs +49 -0
- data/core/src/services/swift/docs.md +52 -0
- data/core/src/services/swift/error.rs +90 -0
- data/core/src/services/swift/lister.rs +119 -0
- data/core/src/services/swift/mod.rs +38 -0
- data/core/src/services/swift/writer.rs +53 -0
- data/core/src/services/tikv/backend.rs +237 -0
- data/core/src/services/tikv/config.rs +52 -0
- data/core/src/services/tikv/docs.md +43 -0
- data/core/src/services/tikv/mod.rs +24 -0
- data/core/src/services/upyun/backend.rs +317 -0
- data/core/src/services/upyun/config.rs +51 -0
- data/core/src/services/upyun/core.rs +521 -0
- data/core/src/services/upyun/delete.rs +50 -0
- data/core/src/services/upyun/docs.md +51 -0
- data/core/src/services/upyun/error.rs +97 -0
- data/core/src/services/upyun/lister.rs +101 -0
- data/core/src/services/upyun/mod.rs +38 -0
- data/core/src/services/upyun/writer.rs +127 -0
- data/core/src/services/vercel_artifacts/backend.rs +99 -0
- data/core/src/services/vercel_artifacts/builder.rs +117 -0
- data/core/src/services/vercel_artifacts/config.rs +39 -0
- data/core/src/services/vercel_artifacts/core.rs +112 -0
- data/core/src/services/vercel_artifacts/docs.md +40 -0
- data/core/src/services/vercel_artifacts/error.rs +50 -0
- data/core/src/services/vercel_artifacts/mod.rs +36 -0
- data/core/src/services/vercel_artifacts/writer.rs +58 -0
- data/core/src/services/vercel_blob/backend.rs +251 -0
- data/core/src/services/vercel_blob/config.rs +45 -0
- data/core/src/services/vercel_blob/core.rs +449 -0
- data/core/src/services/vercel_blob/delete.rs +38 -0
- data/core/src/services/vercel_blob/docs.md +45 -0
- data/core/src/services/vercel_blob/error.rs +110 -0
- data/core/src/services/vercel_blob/lister.rs +69 -0
- data/core/src/services/vercel_blob/mod.rs +38 -0
- data/core/src/services/vercel_blob/writer.rs +143 -0
- data/core/src/services/webdav/backend.rs +318 -0
- data/core/src/services/webdav/config.rs +53 -0
- data/core/src/services/webdav/core.rs +859 -0
- data/core/src/services/webdav/delete.rs +47 -0
- data/core/src/services/webdav/docs.md +49 -0
- data/core/src/services/webdav/error.rs +53 -0
- data/core/src/services/webdav/lister.rs +106 -0
- data/core/src/services/webdav/mod.rs +38 -0
- data/core/src/services/webdav/writer.rs +56 -0
- data/core/src/services/webhdfs/backend.rs +376 -0
- data/core/src/services/webhdfs/config.rs +52 -0
- data/core/src/services/webhdfs/core.rs +398 -0
- data/core/src/services/webhdfs/delete.rs +46 -0
- data/core/src/services/webhdfs/docs.md +90 -0
- data/core/src/services/webhdfs/error.rs +126 -0
- data/core/src/services/webhdfs/lister.rs +130 -0
- data/core/src/services/webhdfs/message.rs +249 -0
- data/core/src/services/webhdfs/mod.rs +41 -0
- data/core/src/services/webhdfs/writer.rs +177 -0
- data/core/src/services/yandex_disk/backend.rs +267 -0
- data/core/src/services/yandex_disk/config.rs +45 -0
- data/core/src/services/yandex_disk/core.rs +340 -0
- data/core/src/services/yandex_disk/delete.rs +54 -0
- data/core/src/services/yandex_disk/docs.md +45 -0
- data/core/src/services/yandex_disk/error.rs +104 -0
- data/core/src/services/yandex_disk/lister.rs +113 -0
- data/core/src/services/yandex_disk/mod.rs +38 -0
- data/core/src/services/yandex_disk/writer.rs +52 -0
- data/core/src/types/buffer.rs +991 -0
- data/core/src/types/builder.rs +152 -0
- data/core/src/types/capability.rs +209 -0
- data/core/src/types/context/mod.rs +22 -0
- data/core/src/types/context/read.rs +231 -0
- data/core/src/types/context/write.rs +441 -0
- data/core/src/types/delete/deleter.rs +220 -0
- data/core/src/types/delete/futures_delete_sink.rs +176 -0
- data/core/src/types/delete/input.rs +97 -0
- data/core/src/types/delete/mod.rs +26 -0
- data/core/src/types/entry.rs +69 -0
- data/core/src/types/error.rs +570 -0
- data/core/src/types/execute/api.rs +110 -0
- data/core/src/types/execute/executor.rs +96 -0
- data/core/src/types/execute/executors/mod.rs +27 -0
- data/core/src/types/execute/executors/tokio_executor.rs +60 -0
- data/core/src/types/execute/mod.rs +25 -0
- data/core/src/types/list.rs +137 -0
- data/core/src/types/metadata.rs +436 -0
- data/core/src/types/mod.rs +72 -0
- data/core/src/types/mode.rs +68 -0
- data/core/src/types/operator/builder.rs +535 -0
- data/core/src/types/operator/info.rs +63 -0
- data/core/src/types/operator/mod.rs +33 -0
- data/core/src/types/operator/operator.rs +2236 -0
- data/core/src/types/operator/operator_futures.rs +1430 -0
- data/core/src/types/operator/registry.rs +129 -0
- data/core/src/types/options.rs +548 -0
- data/core/src/types/read/buffer_stream.rs +273 -0
- data/core/src/types/read/futures_async_reader.rs +289 -0
- data/core/src/types/read/futures_bytes_stream.rs +157 -0
- data/core/src/types/read/mod.rs +29 -0
- data/core/src/types/read/reader.rs +604 -0
- data/core/src/types/scheme.rs +475 -0
- data/core/src/types/write/buffer_sink.rs +188 -0
- data/core/src/types/write/futures_async_writer.rs +136 -0
- data/core/src/types/write/futures_bytes_sink.rs +103 -0
- data/core/src/types/write/mod.rs +26 -0
- data/core/src/types/write/writer.rs +411 -0
- data/core/tests/behavior/README.md +77 -0
- data/core/tests/behavior/async_copy.rs +314 -0
- data/core/tests/behavior/async_create_dir.rs +53 -0
- data/core/tests/behavior/async_delete.rs +354 -0
- data/core/tests/behavior/async_list.rs +739 -0
- data/core/tests/behavior/async_presign.rs +175 -0
- data/core/tests/behavior/async_read.rs +871 -0
- data/core/tests/behavior/async_rename.rs +210 -0
- data/core/tests/behavior/async_stat.rs +628 -0
- data/core/tests/behavior/async_write.rs +819 -0
- data/core/tests/behavior/main.rs +78 -0
- data/core/tests/behavior/utils.rs +187 -0
- data/core/tests/data/normal_dir/.gitkeep +0 -0
- data/core/tests/data/normal_file.txt +1041 -0
- data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
- data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
- data/core/users.md +13 -0
- data/extconf.rb +24 -0
- data/lib/opendal.rb +25 -0
- data/lib/opendal_ruby/entry.rb +35 -0
- data/lib/opendal_ruby/io.rb +70 -0
- data/lib/opendal_ruby/metadata.rb +44 -0
- data/lib/opendal_ruby/operator.rb +29 -0
- data/lib/opendal_ruby/operator_info.rb +26 -0
- data/src/capability.rs +146 -0
- data/src/io.rs +464 -0
- data/src/lib.rs +63 -0
- data/src/lister.rs +141 -0
- data/src/metadata.rs +111 -0
- data/src/middlewares.rs +174 -0
- data/src/operator.rs +310 -0
- data/src/operator_info.rs +83 -0
- data/test/blocking_op_test.rb +112 -0
- data/test/capability_test.rb +42 -0
- data/test/io_test.rb +172 -0
- data/test/lister_test.rb +77 -0
- data/test/metadata_test.rb +78 -0
- data/test/middlewares_test.rb +46 -0
- data/test/operator_info_test.rb +35 -0
- data/test/test_helper.rb +36 -0
- metadata +857 -0
|
@@ -0,0 +1,856 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use std::fmt::Debug;
|
|
19
|
+
use std::future::ready;
|
|
20
|
+
use std::hash::Hash;
|
|
21
|
+
use std::hash::Hasher;
|
|
22
|
+
use std::mem;
|
|
23
|
+
use std::sync::Arc;
|
|
24
|
+
|
|
25
|
+
use futures::Future;
|
|
26
|
+
|
|
27
|
+
use crate::raw::*;
|
|
28
|
+
use crate::*;
|
|
29
|
+
|
|
30
|
+
/// Underlying trait of all backends for implementers.
|
|
31
|
+
///
|
|
32
|
+
/// The actual data access of storage service happens in Accessor layer.
|
|
33
|
+
/// Every storage supported by OpenDAL must implement [`Access`] but not all
|
|
34
|
+
/// methods of [`Access`] will be implemented according to how the storage service is.
|
|
35
|
+
///
|
|
36
|
+
/// For example, user can not modify the content from one HTTP file server directly.
|
|
37
|
+
/// So [`Http`][crate::services::Http] implements and provides only read related actions.
|
|
38
|
+
///
|
|
39
|
+
/// [`Access`] gives default implementation for all methods which will raise [`ErrorKind::Unsupported`] error.
|
|
40
|
+
/// And what action this [`Access`] supports will be pointed out in [`AccessorInfo`].
|
|
41
|
+
///
|
|
42
|
+
/// # Note
|
|
43
|
+
///
|
|
44
|
+
/// Visit [`internals`][crate::docs::internals] for more tutorials.
|
|
45
|
+
///
|
|
46
|
+
/// # Operations
|
|
47
|
+
///
|
|
48
|
+
/// - Path in args will all be normalized into the same style, services
|
|
49
|
+
/// should handle them based on services' requirement.
|
|
50
|
+
/// - Path that ends with `/` means it's Dir, otherwise, it's File.
|
|
51
|
+
/// - Root dir is `/`
|
|
52
|
+
/// - Path will never be empty.
|
|
53
|
+
/// - Operations without capability requirement like `metadata`, `create` are
|
|
54
|
+
/// basic operations.
|
|
55
|
+
/// - All services must implement them.
|
|
56
|
+
/// - Use `unimplemented!()` if not implemented or can't implement.
|
|
57
|
+
/// - Operations with capability requirement like `presign` are optional operations.
|
|
58
|
+
/// - Services can implement them based on services capabilities.
|
|
59
|
+
/// - The default implementation should return [`ErrorKind::Unsupported`].
|
|
60
|
+
pub trait Access: Send + Sync + Debug + Unpin + 'static {
|
|
61
|
+
/// Reader is the associated reader returned in `read` operation.
|
|
62
|
+
type Reader: oio::Read;
|
|
63
|
+
/// Writer is the associated writer returned in `write` operation.
|
|
64
|
+
type Writer: oio::Write;
|
|
65
|
+
/// Lister is the associated lister returned in `list` operation.
|
|
66
|
+
type Lister: oio::List;
|
|
67
|
+
/// Deleter is the associated deleter returned in `delete` operation.
|
|
68
|
+
type Deleter: oio::Delete;
|
|
69
|
+
|
|
70
|
+
/// Invoke the `info` operation to get metadata of accessor.
|
|
71
|
+
///
|
|
72
|
+
/// # Notes
|
|
73
|
+
///
|
|
74
|
+
/// This function is required to be implemented.
|
|
75
|
+
///
|
|
76
|
+
/// By returning AccessorInfo, underlying services can declare
|
|
77
|
+
/// some useful information about itself.
|
|
78
|
+
///
|
|
79
|
+
/// - scheme: declare the scheme of backend.
|
|
80
|
+
/// - capabilities: declare the capabilities of current backend.
|
|
81
|
+
fn info(&self) -> Arc<AccessorInfo>;
|
|
82
|
+
|
|
83
|
+
/// Invoke the `create` operation on the specified path
|
|
84
|
+
///
|
|
85
|
+
/// Require [`Capability::create_dir`]
|
|
86
|
+
///
|
|
87
|
+
/// # Behavior
|
|
88
|
+
///
|
|
89
|
+
/// - Input path MUST match with EntryMode, DON'T NEED to check mode.
|
|
90
|
+
/// - Create on existing dir SHOULD succeed.
|
|
91
|
+
fn create_dir(
|
|
92
|
+
&self,
|
|
93
|
+
path: &str,
|
|
94
|
+
args: OpCreateDir,
|
|
95
|
+
) -> impl Future<Output = Result<RpCreateDir>> + MaybeSend {
|
|
96
|
+
let (_, _) = (path, args);
|
|
97
|
+
|
|
98
|
+
ready(Err(Error::new(
|
|
99
|
+
ErrorKind::Unsupported,
|
|
100
|
+
"operation is not supported",
|
|
101
|
+
)))
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/// Invoke the `stat` operation on the specified path.
|
|
105
|
+
///
|
|
106
|
+
/// Require [`Capability::stat`]
|
|
107
|
+
///
|
|
108
|
+
/// # Behavior
|
|
109
|
+
///
|
|
110
|
+
/// - `stat` empty path means stat backend's root path.
|
|
111
|
+
/// - `stat` a path endswith "/" means stating a dir.
|
|
112
|
+
/// - `mode` and `content_length` must be set.
|
|
113
|
+
fn stat(&self, path: &str, args: OpStat) -> impl Future<Output = Result<RpStat>> + MaybeSend {
|
|
114
|
+
let (_, _) = (path, args);
|
|
115
|
+
|
|
116
|
+
ready(Err(Error::new(
|
|
117
|
+
ErrorKind::Unsupported,
|
|
118
|
+
"operation is not supported",
|
|
119
|
+
)))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/// Invoke the `read` operation on the specified path, returns a
|
|
123
|
+
/// [`Reader`][crate::Reader] if operate successful.
|
|
124
|
+
///
|
|
125
|
+
/// Require [`Capability::read`]
|
|
126
|
+
///
|
|
127
|
+
/// # Behavior
|
|
128
|
+
///
|
|
129
|
+
/// - Input path MUST be file path, DON'T NEED to check mode.
|
|
130
|
+
/// - The returning content length may be smaller than the range specified.
|
|
131
|
+
fn read(
|
|
132
|
+
&self,
|
|
133
|
+
path: &str,
|
|
134
|
+
args: OpRead,
|
|
135
|
+
) -> impl Future<Output = Result<(RpRead, Self::Reader)>> + MaybeSend {
|
|
136
|
+
let (_, _) = (path, args);
|
|
137
|
+
|
|
138
|
+
ready(Err(Error::new(
|
|
139
|
+
ErrorKind::Unsupported,
|
|
140
|
+
"operation is not supported",
|
|
141
|
+
)))
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/// Invoke the `write` operation on the specified path, returns a
|
|
145
|
+
/// written size if operate successful.
|
|
146
|
+
///
|
|
147
|
+
/// Require [`Capability::write`]
|
|
148
|
+
///
|
|
149
|
+
/// # Behavior
|
|
150
|
+
///
|
|
151
|
+
/// - Input path MUST be file path, DON'T NEED to check mode.
|
|
152
|
+
fn write(
|
|
153
|
+
&self,
|
|
154
|
+
path: &str,
|
|
155
|
+
args: OpWrite,
|
|
156
|
+
) -> impl Future<Output = Result<(RpWrite, Self::Writer)>> + MaybeSend {
|
|
157
|
+
let (_, _) = (path, args);
|
|
158
|
+
|
|
159
|
+
ready(Err(Error::new(
|
|
160
|
+
ErrorKind::Unsupported,
|
|
161
|
+
"operation is not supported",
|
|
162
|
+
)))
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/// Invoke the `delete` operation on the specified path.
|
|
166
|
+
///
|
|
167
|
+
/// Require [`Capability::delete`]
|
|
168
|
+
///
|
|
169
|
+
/// # Behavior
|
|
170
|
+
///
|
|
171
|
+
/// - `delete` is an idempotent operation, it's safe to call `Delete` on the same path multiple times.
|
|
172
|
+
/// - `delete` SHOULD return `Ok(())` if the path is deleted successfully or not exist.
|
|
173
|
+
fn delete(&self) -> impl Future<Output = Result<(RpDelete, Self::Deleter)>> + MaybeSend {
|
|
174
|
+
ready(Err(Error::new(
|
|
175
|
+
ErrorKind::Unsupported,
|
|
176
|
+
"operation is not supported",
|
|
177
|
+
)))
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/// Invoke the `list` operation on the specified path.
|
|
181
|
+
///
|
|
182
|
+
/// Require [`Capability::list`]
|
|
183
|
+
///
|
|
184
|
+
/// # Behavior
|
|
185
|
+
///
|
|
186
|
+
/// - Input path MUST be dir path, DON'T NEED to check mode.
|
|
187
|
+
/// - List non-exist dir should return Empty.
|
|
188
|
+
fn list(
|
|
189
|
+
&self,
|
|
190
|
+
path: &str,
|
|
191
|
+
args: OpList,
|
|
192
|
+
) -> impl Future<Output = Result<(RpList, Self::Lister)>> + MaybeSend {
|
|
193
|
+
let (_, _) = (path, args);
|
|
194
|
+
|
|
195
|
+
ready(Err(Error::new(
|
|
196
|
+
ErrorKind::Unsupported,
|
|
197
|
+
"operation is not supported",
|
|
198
|
+
)))
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/// Invoke the `copy` operation on the specified `from` path and `to` path.
|
|
202
|
+
///
|
|
203
|
+
/// Require [Capability::copy]
|
|
204
|
+
///
|
|
205
|
+
/// # Behaviour
|
|
206
|
+
///
|
|
207
|
+
/// - `from` and `to` MUST be file path, DON'T NEED to check mode.
|
|
208
|
+
/// - Copy on existing file SHOULD succeed.
|
|
209
|
+
/// - Copy on existing file SHOULD overwrite and truncate.
|
|
210
|
+
fn copy(
|
|
211
|
+
&self,
|
|
212
|
+
from: &str,
|
|
213
|
+
to: &str,
|
|
214
|
+
args: OpCopy,
|
|
215
|
+
) -> impl Future<Output = Result<RpCopy>> + MaybeSend {
|
|
216
|
+
let (_, _, _) = (from, to, args);
|
|
217
|
+
|
|
218
|
+
ready(Err(Error::new(
|
|
219
|
+
ErrorKind::Unsupported,
|
|
220
|
+
"operation is not supported",
|
|
221
|
+
)))
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/// Invoke the `rename` operation on the specified `from` path and `to` path.
|
|
225
|
+
///
|
|
226
|
+
/// Require [Capability::rename]
|
|
227
|
+
fn rename(
|
|
228
|
+
&self,
|
|
229
|
+
from: &str,
|
|
230
|
+
to: &str,
|
|
231
|
+
args: OpRename,
|
|
232
|
+
) -> impl Future<Output = Result<RpRename>> + MaybeSend {
|
|
233
|
+
let (_, _, _) = (from, to, args);
|
|
234
|
+
|
|
235
|
+
ready(Err(Error::new(
|
|
236
|
+
ErrorKind::Unsupported,
|
|
237
|
+
"operation is not supported",
|
|
238
|
+
)))
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/// Invoke the `presign` operation on the specified path.
|
|
242
|
+
///
|
|
243
|
+
/// Require [`Capability::presign`]
|
|
244
|
+
///
|
|
245
|
+
/// # Behavior
|
|
246
|
+
///
|
|
247
|
+
/// - This API is optional, return [`std::io::ErrorKind::Unsupported`] if not supported.
|
|
248
|
+
fn presign(
|
|
249
|
+
&self,
|
|
250
|
+
path: &str,
|
|
251
|
+
args: OpPresign,
|
|
252
|
+
) -> impl Future<Output = Result<RpPresign>> + MaybeSend {
|
|
253
|
+
let (_, _) = (path, args);
|
|
254
|
+
|
|
255
|
+
ready(Err(Error::new(
|
|
256
|
+
ErrorKind::Unsupported,
|
|
257
|
+
"operation is not supported",
|
|
258
|
+
)))
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/// `AccessDyn` is the dyn version of [`Access`] make it possible to use as
|
|
263
|
+
/// `Box<dyn AccessDyn>`.
|
|
264
|
+
pub trait AccessDyn: Send + Sync + Debug + Unpin {
|
|
265
|
+
/// Dyn version of [`Accessor::info`]
|
|
266
|
+
fn info_dyn(&self) -> Arc<AccessorInfo>;
|
|
267
|
+
/// Dyn version of [`Accessor::create_dir`]
|
|
268
|
+
fn create_dir_dyn<'a>(
|
|
269
|
+
&'a self,
|
|
270
|
+
path: &'a str,
|
|
271
|
+
args: OpCreateDir,
|
|
272
|
+
) -> BoxedFuture<'a, Result<RpCreateDir>>;
|
|
273
|
+
/// Dyn version of [`Accessor::stat`]
|
|
274
|
+
fn stat_dyn<'a>(&'a self, path: &'a str, args: OpStat) -> BoxedFuture<'a, Result<RpStat>>;
|
|
275
|
+
/// Dyn version of [`Accessor::read`]
|
|
276
|
+
fn read_dyn<'a>(
|
|
277
|
+
&'a self,
|
|
278
|
+
path: &'a str,
|
|
279
|
+
args: OpRead,
|
|
280
|
+
) -> BoxedFuture<'a, Result<(RpRead, oio::Reader)>>;
|
|
281
|
+
/// Dyn version of [`Accessor::write`]
|
|
282
|
+
fn write_dyn<'a>(
|
|
283
|
+
&'a self,
|
|
284
|
+
path: &'a str,
|
|
285
|
+
args: OpWrite,
|
|
286
|
+
) -> BoxedFuture<'a, Result<(RpWrite, oio::Writer)>>;
|
|
287
|
+
/// Dyn version of [`Accessor::delete`]
|
|
288
|
+
fn delete_dyn(&self) -> BoxedFuture<'_, Result<(RpDelete, oio::Deleter)>>;
|
|
289
|
+
/// Dyn version of [`Accessor::list`]
|
|
290
|
+
fn list_dyn<'a>(
|
|
291
|
+
&'a self,
|
|
292
|
+
path: &'a str,
|
|
293
|
+
args: OpList,
|
|
294
|
+
) -> BoxedFuture<'a, Result<(RpList, oio::Lister)>>;
|
|
295
|
+
/// Dyn version of [`Accessor::copy`]
|
|
296
|
+
fn copy_dyn<'a>(
|
|
297
|
+
&'a self,
|
|
298
|
+
from: &'a str,
|
|
299
|
+
to: &'a str,
|
|
300
|
+
args: OpCopy,
|
|
301
|
+
) -> BoxedFuture<'a, Result<RpCopy>>;
|
|
302
|
+
/// Dyn version of [`Accessor::rename`]
|
|
303
|
+
fn rename_dyn<'a>(
|
|
304
|
+
&'a self,
|
|
305
|
+
from: &'a str,
|
|
306
|
+
to: &'a str,
|
|
307
|
+
args: OpRename,
|
|
308
|
+
) -> BoxedFuture<'a, Result<RpRename>>;
|
|
309
|
+
/// Dyn version of [`Accessor::presign`]
|
|
310
|
+
fn presign_dyn<'a>(
|
|
311
|
+
&'a self,
|
|
312
|
+
path: &'a str,
|
|
313
|
+
args: OpPresign,
|
|
314
|
+
) -> BoxedFuture<'a, Result<RpPresign>>;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
impl<A: ?Sized> AccessDyn for A
|
|
318
|
+
where
|
|
319
|
+
A: Access<
|
|
320
|
+
Reader = oio::Reader,
|
|
321
|
+
Writer = oio::Writer,
|
|
322
|
+
Lister = oio::Lister,
|
|
323
|
+
Deleter = oio::Deleter,
|
|
324
|
+
>,
|
|
325
|
+
{
|
|
326
|
+
fn info_dyn(&self) -> Arc<AccessorInfo> {
|
|
327
|
+
self.info()
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
fn create_dir_dyn<'a>(
|
|
331
|
+
&'a self,
|
|
332
|
+
path: &'a str,
|
|
333
|
+
args: OpCreateDir,
|
|
334
|
+
) -> BoxedFuture<'a, Result<RpCreateDir>> {
|
|
335
|
+
Box::pin(self.create_dir(path, args))
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
fn stat_dyn<'a>(&'a self, path: &'a str, args: OpStat) -> BoxedFuture<'a, Result<RpStat>> {
|
|
339
|
+
Box::pin(self.stat(path, args))
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
fn read_dyn<'a>(
|
|
343
|
+
&'a self,
|
|
344
|
+
path: &'a str,
|
|
345
|
+
args: OpRead,
|
|
346
|
+
) -> BoxedFuture<'a, Result<(RpRead, oio::Reader)>> {
|
|
347
|
+
Box::pin(self.read(path, args))
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
fn write_dyn<'a>(
|
|
351
|
+
&'a self,
|
|
352
|
+
path: &'a str,
|
|
353
|
+
args: OpWrite,
|
|
354
|
+
) -> BoxedFuture<'a, Result<(RpWrite, oio::Writer)>> {
|
|
355
|
+
Box::pin(self.write(path, args))
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
fn delete_dyn(&self) -> BoxedFuture<'_, Result<(RpDelete, oio::Deleter)>> {
|
|
359
|
+
Box::pin(self.delete())
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
fn list_dyn<'a>(
|
|
363
|
+
&'a self,
|
|
364
|
+
path: &'a str,
|
|
365
|
+
args: OpList,
|
|
366
|
+
) -> BoxedFuture<'a, Result<(RpList, oio::Lister)>> {
|
|
367
|
+
Box::pin(self.list(path, args))
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
fn copy_dyn<'a>(
|
|
371
|
+
&'a self,
|
|
372
|
+
from: &'a str,
|
|
373
|
+
to: &'a str,
|
|
374
|
+
args: OpCopy,
|
|
375
|
+
) -> BoxedFuture<'a, Result<RpCopy>> {
|
|
376
|
+
Box::pin(self.copy(from, to, args))
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
fn rename_dyn<'a>(
|
|
380
|
+
&'a self,
|
|
381
|
+
from: &'a str,
|
|
382
|
+
to: &'a str,
|
|
383
|
+
args: OpRename,
|
|
384
|
+
) -> BoxedFuture<'a, Result<RpRename>> {
|
|
385
|
+
Box::pin(self.rename(from, to, args))
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
fn presign_dyn<'a>(
|
|
389
|
+
&'a self,
|
|
390
|
+
path: &'a str,
|
|
391
|
+
args: OpPresign,
|
|
392
|
+
) -> BoxedFuture<'a, Result<RpPresign>> {
|
|
393
|
+
Box::pin(self.presign(path, args))
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
impl Access for dyn AccessDyn {
|
|
398
|
+
type Reader = oio::Reader;
|
|
399
|
+
type Writer = oio::Writer;
|
|
400
|
+
type Deleter = oio::Deleter;
|
|
401
|
+
type Lister = oio::Lister;
|
|
402
|
+
|
|
403
|
+
fn info(&self) -> Arc<AccessorInfo> {
|
|
404
|
+
self.info_dyn()
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
async fn create_dir(&self, path: &str, args: OpCreateDir) -> Result<RpCreateDir> {
|
|
408
|
+
self.create_dir_dyn(path, args).await
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
async fn stat(&self, path: &str, args: OpStat) -> Result<RpStat> {
|
|
412
|
+
self.stat_dyn(path, args).await
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
|
|
416
|
+
self.read_dyn(path, args).await
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> {
|
|
420
|
+
self.write_dyn(path, args).await
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> {
|
|
424
|
+
self.delete_dyn().await
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> {
|
|
428
|
+
self.list_dyn(path, args).await
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result<RpCopy> {
|
|
432
|
+
self.copy_dyn(from, to, args).await
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
async fn rename(&self, from: &str, to: &str, args: OpRename) -> Result<RpRename> {
|
|
436
|
+
self.rename_dyn(from, to, args).await
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
async fn presign(&self, path: &str, args: OpPresign) -> Result<RpPresign> {
|
|
440
|
+
self.presign_dyn(path, args).await
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/// Dummy implementation of accessor.
|
|
445
|
+
impl Access for () {
|
|
446
|
+
type Reader = ();
|
|
447
|
+
type Writer = ();
|
|
448
|
+
type Lister = ();
|
|
449
|
+
type Deleter = ();
|
|
450
|
+
|
|
451
|
+
fn info(&self) -> Arc<AccessorInfo> {
|
|
452
|
+
let ai = AccessorInfo::default();
|
|
453
|
+
ai.set_scheme("dummy")
|
|
454
|
+
.set_root("")
|
|
455
|
+
.set_name("dummy")
|
|
456
|
+
.set_native_capability(Capability::default());
|
|
457
|
+
ai.into()
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/// All functions in `Accessor` only requires `&self`, so it's safe to implement
|
|
462
|
+
/// `Accessor` for `Arc<impl Access>`.
|
|
463
|
+
// If we use async fn directly, some weird higher rank trait bound error (`Send`/`Accessor` impl not general enough) will happen.
|
|
464
|
+
// Probably related to https://github.com/rust-lang/rust/issues/96865
|
|
465
|
+
#[allow(clippy::manual_async_fn)]
|
|
466
|
+
impl<T: Access + ?Sized> Access for Arc<T> {
|
|
467
|
+
type Reader = T::Reader;
|
|
468
|
+
type Writer = T::Writer;
|
|
469
|
+
type Lister = T::Lister;
|
|
470
|
+
type Deleter = T::Deleter;
|
|
471
|
+
|
|
472
|
+
fn info(&self) -> Arc<AccessorInfo> {
|
|
473
|
+
self.as_ref().info()
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
fn create_dir(
|
|
477
|
+
&self,
|
|
478
|
+
path: &str,
|
|
479
|
+
args: OpCreateDir,
|
|
480
|
+
) -> impl Future<Output = Result<RpCreateDir>> + MaybeSend {
|
|
481
|
+
async move { self.as_ref().create_dir(path, args).await }
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
fn stat(&self, path: &str, args: OpStat) -> impl Future<Output = Result<RpStat>> + MaybeSend {
|
|
485
|
+
async move { self.as_ref().stat(path, args).await }
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
fn read(
|
|
489
|
+
&self,
|
|
490
|
+
path: &str,
|
|
491
|
+
args: OpRead,
|
|
492
|
+
) -> impl Future<Output = Result<(RpRead, Self::Reader)>> + MaybeSend {
|
|
493
|
+
async move { self.as_ref().read(path, args).await }
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
fn write(
|
|
497
|
+
&self,
|
|
498
|
+
path: &str,
|
|
499
|
+
args: OpWrite,
|
|
500
|
+
) -> impl Future<Output = Result<(RpWrite, Self::Writer)>> + MaybeSend {
|
|
501
|
+
async move { self.as_ref().write(path, args).await }
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
fn delete(&self) -> impl Future<Output = Result<(RpDelete, Self::Deleter)>> + MaybeSend {
|
|
505
|
+
async move { self.as_ref().delete().await }
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
fn list(
|
|
509
|
+
&self,
|
|
510
|
+
path: &str,
|
|
511
|
+
args: OpList,
|
|
512
|
+
) -> impl Future<Output = Result<(RpList, Self::Lister)>> + MaybeSend {
|
|
513
|
+
async move { self.as_ref().list(path, args).await }
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
fn copy(
|
|
517
|
+
&self,
|
|
518
|
+
from: &str,
|
|
519
|
+
to: &str,
|
|
520
|
+
args: OpCopy,
|
|
521
|
+
) -> impl Future<Output = Result<RpCopy>> + MaybeSend {
|
|
522
|
+
async move { self.as_ref().copy(from, to, args).await }
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
fn rename(
|
|
526
|
+
&self,
|
|
527
|
+
from: &str,
|
|
528
|
+
to: &str,
|
|
529
|
+
args: OpRename,
|
|
530
|
+
) -> impl Future<Output = Result<RpRename>> + MaybeSend {
|
|
531
|
+
async move { self.as_ref().rename(from, to, args).await }
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
fn presign(
|
|
535
|
+
&self,
|
|
536
|
+
path: &str,
|
|
537
|
+
args: OpPresign,
|
|
538
|
+
) -> impl Future<Output = Result<RpPresign>> + MaybeSend {
|
|
539
|
+
async move { self.as_ref().presign(path, args).await }
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/// Accessor is the type erased accessor with `Arc<dyn Accessor>`.
|
|
544
|
+
pub type Accessor = Arc<dyn AccessDyn>;
|
|
545
|
+
|
|
546
|
+
#[derive(Debug)]
|
|
547
|
+
struct AccessorInfoInner {
|
|
548
|
+
scheme: &'static str,
|
|
549
|
+
root: Arc<str>,
|
|
550
|
+
name: Arc<str>,
|
|
551
|
+
|
|
552
|
+
native_capability: Capability,
|
|
553
|
+
full_capability: Capability,
|
|
554
|
+
|
|
555
|
+
http_client: HttpClient,
|
|
556
|
+
executor: Executor,
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
impl Default for AccessorInfoInner {
|
|
560
|
+
fn default() -> Self {
|
|
561
|
+
Self {
|
|
562
|
+
scheme: "unknown",
|
|
563
|
+
root: Arc::from(""),
|
|
564
|
+
name: Arc::from(""),
|
|
565
|
+
native_capability: Capability::default(),
|
|
566
|
+
full_capability: Capability::default(),
|
|
567
|
+
http_client: HttpClient::default(),
|
|
568
|
+
executor: Executor::default(),
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/// Info for the accessor. Users can use this struct to retrieve information about the underlying backend.
|
|
574
|
+
///
|
|
575
|
+
/// This struct is intentionally not implemented with `Clone` to ensure that all accesses
|
|
576
|
+
/// within the same operator, access layers, and services use the same instance of `AccessorInfo`.
|
|
577
|
+
/// This is especially important for `HttpClient` and `Executor`.
|
|
578
|
+
///
|
|
579
|
+
/// ## Panic Safety
|
|
580
|
+
///
|
|
581
|
+
/// All methods provided by `AccessorInfo` will safely handle lock poisoning scenarios.
|
|
582
|
+
/// If the inner `RwLock` is poisoned (which happens when another thread panicked while holding
|
|
583
|
+
/// the write lock), this method will gracefully continue execution.
|
|
584
|
+
///
|
|
585
|
+
/// - For read operations, the method will return the current state.
|
|
586
|
+
/// - For write operations, the method will do nothing.
|
|
587
|
+
///
|
|
588
|
+
/// ## Maintain Notes
|
|
589
|
+
///
|
|
590
|
+
/// We are using `std::sync::RwLock` to provide thread-safe access to the inner data.
|
|
591
|
+
///
|
|
592
|
+
/// I have performed [the bench across different arc-swap alike crates](https://github.com/krdln/arc-swap-benches):
|
|
593
|
+
///
|
|
594
|
+
/// ```txt
|
|
595
|
+
/// test arcswap ... bench: 14.85 ns/iter (+/- 0.33)
|
|
596
|
+
/// test arcswap_full ... bench: 128.27 ns/iter (+/- 4.30)
|
|
597
|
+
/// test baseline ... bench: 11.33 ns/iter (+/- 0.76)
|
|
598
|
+
/// test mutex_4 ... bench: 296.73 ns/iter (+/- 49.96)
|
|
599
|
+
/// test mutex_unconteded ... bench: 13.26 ns/iter (+/- 0.56)
|
|
600
|
+
/// test rwlock_fast_4 ... bench: 201.60 ns/iter (+/- 7.47)
|
|
601
|
+
/// test rwlock_fast_uncontended ... bench: 12.77 ns/iter (+/- 0.37)
|
|
602
|
+
/// test rwlock_parking_4 ... bench: 232.02 ns/iter (+/- 11.14)
|
|
603
|
+
/// test rwlock_parking_uncontended ... bench: 13.18 ns/iter (+/- 0.39)
|
|
604
|
+
/// test rwlock_std_4 ... bench: 219.56 ns/iter (+/- 5.56)
|
|
605
|
+
/// test rwlock_std_uncontended ... bench: 13.55 ns/iter (+/- 0.33)
|
|
606
|
+
/// ```
|
|
607
|
+
///
|
|
608
|
+
/// The results show that as long as there aren't too many uncontended accesses, `RwLock` is the
|
|
609
|
+
/// best choice, allowing for fast access and the ability to modify partial data without cloning
|
|
610
|
+
/// everything.
|
|
611
|
+
///
|
|
612
|
+
/// And it's true: we only update and modify the internal data in a few instances, such as when
|
|
613
|
+
/// building an operator or applying new layers.
|
|
614
|
+
#[derive(Debug, Default)]
|
|
615
|
+
pub struct AccessorInfo {
|
|
616
|
+
inner: std::sync::RwLock<AccessorInfoInner>,
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
impl PartialEq for AccessorInfo {
|
|
620
|
+
fn eq(&self, other: &Self) -> bool {
|
|
621
|
+
self.scheme() == other.scheme()
|
|
622
|
+
&& self.root() == other.root()
|
|
623
|
+
&& self.name() == other.name()
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
impl Eq for AccessorInfo {}
|
|
628
|
+
|
|
629
|
+
impl Hash for AccessorInfo {
|
|
630
|
+
fn hash<H: Hasher>(&self, state: &mut H) {
|
|
631
|
+
self.scheme().hash(state);
|
|
632
|
+
self.root().hash(state);
|
|
633
|
+
self.name().hash(state);
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
impl AccessorInfo {
|
|
638
|
+
/// Scheme of backend.
|
|
639
|
+
///
|
|
640
|
+
/// # Panic Safety
|
|
641
|
+
///
|
|
642
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
643
|
+
/// this method will gracefully continue execution by simply returning the current scheme.
|
|
644
|
+
pub fn scheme(&self) -> &'static str {
|
|
645
|
+
match self.inner.read() {
|
|
646
|
+
Ok(v) => v.scheme,
|
|
647
|
+
Err(err) => err.get_ref().scheme,
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/// Set scheme for backend.
|
|
652
|
+
///
|
|
653
|
+
/// # Panic Safety
|
|
654
|
+
///
|
|
655
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
656
|
+
/// this method will gracefully continue execution by simply skipping the update operation
|
|
657
|
+
/// rather than propagating the panic.
|
|
658
|
+
pub fn set_scheme(&self, scheme: &'static str) -> &Self {
|
|
659
|
+
if let Ok(mut v) = self.inner.write() {
|
|
660
|
+
v.scheme = scheme;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
self
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/// Root of backend, will be in format like `/path/to/dir/`
|
|
667
|
+
///
|
|
668
|
+
/// # Panic Safety
|
|
669
|
+
///
|
|
670
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
671
|
+
/// this method will gracefully continue execution by simply returning the current root.
|
|
672
|
+
pub fn root(&self) -> Arc<str> {
|
|
673
|
+
match self.inner.read() {
|
|
674
|
+
Ok(v) => v.root.clone(),
|
|
675
|
+
Err(err) => err.get_ref().root.clone(),
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/// Set root for backend.
|
|
680
|
+
///
|
|
681
|
+
/// Note: input root must be normalized.
|
|
682
|
+
///
|
|
683
|
+
/// # Panic Safety
|
|
684
|
+
///
|
|
685
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
686
|
+
/// this method will gracefully continue execution by simply skipping the update operation
|
|
687
|
+
/// rather than propagating the panic.
|
|
688
|
+
pub fn set_root(&self, root: &str) -> &Self {
|
|
689
|
+
if let Ok(mut v) = self.inner.write() {
|
|
690
|
+
v.root = Arc::from(root);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
self
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/// Name of backend, could be empty if underlying backend doesn't have namespace concept.
|
|
697
|
+
///
|
|
698
|
+
/// For example:
|
|
699
|
+
///
|
|
700
|
+
/// - `s3` => bucket name
|
|
701
|
+
/// - `azblob` => container name
|
|
702
|
+
/// - `azdfs` => filesystem name
|
|
703
|
+
/// - `azfile` => share name
|
|
704
|
+
///
|
|
705
|
+
/// # Panic Safety
|
|
706
|
+
///
|
|
707
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
708
|
+
/// this method will gracefully continue execution by simply returning the current scheme.
|
|
709
|
+
pub fn name(&self) -> Arc<str> {
|
|
710
|
+
match self.inner.read() {
|
|
711
|
+
Ok(v) => v.name.clone(),
|
|
712
|
+
Err(err) => err.get_ref().name.clone(),
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/// Set name of this backend.
|
|
717
|
+
///
|
|
718
|
+
/// # Panic Safety
|
|
719
|
+
///
|
|
720
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
721
|
+
/// this method will gracefully continue execution by simply skipping the update operation
|
|
722
|
+
/// rather than propagating the panic.
|
|
723
|
+
pub fn set_name(&self, name: &str) -> &Self {
|
|
724
|
+
if let Ok(mut v) = self.inner.write() {
|
|
725
|
+
v.name = Arc::from(name)
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
self
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/// Get backend's native capabilities.
|
|
732
|
+
///
|
|
733
|
+
/// # Panic Safety
|
|
734
|
+
///
|
|
735
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
736
|
+
/// this method will gracefully continue execution by simply returning the current native capability.
|
|
737
|
+
pub fn native_capability(&self) -> Capability {
|
|
738
|
+
match self.inner.read() {
|
|
739
|
+
Ok(v) => v.native_capability,
|
|
740
|
+
Err(err) => err.get_ref().native_capability,
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
/// Set native capabilities for service.
|
|
745
|
+
///
|
|
746
|
+
/// # NOTES
|
|
747
|
+
///
|
|
748
|
+
/// Set native capability will also flush the full capability. The only way to change
|
|
749
|
+
/// full_capability is via `update_full_capability`.
|
|
750
|
+
///
|
|
751
|
+
/// # Panic Safety
|
|
752
|
+
///
|
|
753
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
754
|
+
/// this method will gracefully continue execution by simply skipping the update operation
|
|
755
|
+
/// rather than propagating the panic.
|
|
756
|
+
pub fn set_native_capability(&self, capability: Capability) -> &Self {
|
|
757
|
+
if let Ok(mut v) = self.inner.write() {
|
|
758
|
+
v.native_capability = capability;
|
|
759
|
+
v.full_capability = capability;
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
self
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/// Get service's full capabilities.
|
|
766
|
+
///
|
|
767
|
+
/// # Panic Safety
|
|
768
|
+
///
|
|
769
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
770
|
+
/// this method will gracefully continue execution by simply returning the current native capability.
|
|
771
|
+
pub fn full_capability(&self) -> Capability {
|
|
772
|
+
match self.inner.read() {
|
|
773
|
+
Ok(v) => v.full_capability,
|
|
774
|
+
Err(err) => err.get_ref().full_capability,
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/// Update service's full capabilities.
|
|
779
|
+
///
|
|
780
|
+
/// # Panic Safety
|
|
781
|
+
///
|
|
782
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
783
|
+
/// this method will gracefully continue execution by simply skipping the update operation
|
|
784
|
+
/// rather than propagating the panic.
|
|
785
|
+
pub fn update_full_capability(&self, f: impl FnOnce(Capability) -> Capability) -> &Self {
|
|
786
|
+
if let Ok(mut v) = self.inner.write() {
|
|
787
|
+
v.full_capability = f(v.full_capability);
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
self
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/// Get http client from the context.
|
|
794
|
+
///
|
|
795
|
+
/// # Panic Safety
|
|
796
|
+
///
|
|
797
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
798
|
+
/// this method will gracefully continue execution by simply returning the current http client.
|
|
799
|
+
pub fn http_client(&self) -> HttpClient {
|
|
800
|
+
match self.inner.read() {
|
|
801
|
+
Ok(v) => v.http_client.clone(),
|
|
802
|
+
Err(err) => err.get_ref().http_client.clone(),
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
/// Update http client for the context.
|
|
807
|
+
///
|
|
808
|
+
/// # Note
|
|
809
|
+
///
|
|
810
|
+
/// Requests must be forwarded to the old HTTP client after the update. Otherwise, features such as retry, tracing, and metrics may not function properly.
|
|
811
|
+
///
|
|
812
|
+
/// # Panic Safety
|
|
813
|
+
///
|
|
814
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
815
|
+
/// this method will gracefully continue execution by simply skipping the update operation.
|
|
816
|
+
pub fn update_http_client(&self, f: impl FnOnce(HttpClient) -> HttpClient) -> &Self {
|
|
817
|
+
if let Ok(mut v) = self.inner.write() {
|
|
818
|
+
let client = mem::take(&mut v.http_client);
|
|
819
|
+
v.http_client = f(client);
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
self
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/// Get executor from the context.
|
|
826
|
+
///
|
|
827
|
+
/// # Panic Safety
|
|
828
|
+
///
|
|
829
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
830
|
+
/// this method will gracefully continue execution by simply returning the current executor.
|
|
831
|
+
pub fn executor(&self) -> Executor {
|
|
832
|
+
match self.inner.read() {
|
|
833
|
+
Ok(v) => v.executor.clone(),
|
|
834
|
+
Err(err) => err.get_ref().executor.clone(),
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
/// Update executor for the context.
|
|
839
|
+
///
|
|
840
|
+
/// # Note
|
|
841
|
+
///
|
|
842
|
+
/// Tasks must be forwarded to the old executor after the update. Otherwise, features such as retry, timeout, and metrics may not function properly.
|
|
843
|
+
///
|
|
844
|
+
/// # Panic Safety
|
|
845
|
+
///
|
|
846
|
+
/// This method safely handles lock poisoning scenarios. If the inner `RwLock` is poisoned,
|
|
847
|
+
/// this method will gracefully continue execution by simply skipping the update operation.
|
|
848
|
+
pub fn update_executor(&self, f: impl FnOnce(Executor) -> Executor) -> &Self {
|
|
849
|
+
if let Ok(mut v) = self.inner.write() {
|
|
850
|
+
let executor = mem::take(&mut v.executor);
|
|
851
|
+
v.executor = f(executor);
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
self
|
|
855
|
+
}
|
|
856
|
+
}
|