grpc 1.64.0 → 1.65.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Makefile +57 -58
- data/include/grpc/event_engine/event_engine.h +13 -6
- data/include/grpc/impl/channel_arg_names.h +7 -3
- data/include/grpc/module.modulemap +1 -0
- data/include/grpc/passive_listener.h +62 -0
- data/include/grpc/support/log.h +1 -6
- data/include/grpc/support/port_platform.h +3 -0
- data/src/core/channelz/channel_trace.cc +1 -1
- data/src/core/channelz/channel_trace.h +1 -1
- data/src/core/channelz/channelz.cc +3 -3
- data/src/core/channelz/channelz.h +7 -7
- data/src/core/channelz/channelz_registry.cc +4 -3
- data/src/core/client_channel/backup_poller.cc +4 -5
- data/src/core/client_channel/client_channel.cc +1324 -0
- data/src/core/client_channel/client_channel.h +243 -0
- data/src/core/client_channel/client_channel_filter.cc +266 -709
- data/src/core/client_channel/client_channel_filter.h +11 -64
- data/src/core/client_channel/client_channel_internal.h +16 -5
- data/src/core/client_channel/client_channel_plugin.cc +1 -14
- data/src/core/client_channel/client_channel_service_config.h +3 -3
- data/src/core/client_channel/config_selector.cc +1 -1
- data/src/core/client_channel/config_selector.h +1 -1
- data/src/core/client_channel/dynamic_filters.cc +3 -3
- data/src/core/client_channel/dynamic_filters.h +1 -3
- data/src/core/client_channel/load_balanced_call_destination.cc +336 -0
- data/src/core/client_channel/load_balanced_call_destination.h +49 -0
- data/src/core/client_channel/retry_filter.cc +2 -9
- data/src/core/client_channel/retry_filter.h +2 -7
- data/src/core/client_channel/retry_filter_legacy_call_data.cc +65 -72
- data/src/core/client_channel/retry_filter_legacy_call_data.h +0 -2
- data/src/core/client_channel/retry_service_config.cc +4 -5
- data/src/core/client_channel/retry_service_config.h +3 -3
- data/src/core/client_channel/subchannel.cc +220 -112
- data/src/core/client_channel/subchannel.h +31 -18
- data/src/core/client_channel/subchannel_pool_interface.cc +0 -2
- data/src/core/client_channel/subchannel_pool_interface.h +2 -4
- data/src/core/client_channel/subchannel_stream_client.cc +36 -49
- data/src/core/client_channel/subchannel_stream_client.h +2 -4
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +7 -10
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +1 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +7 -0
- data/src/core/ext/filters/census/grpc_context.cc +2 -4
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +8 -15
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h +2 -0
- data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +3 -7
- data/src/core/ext/filters/fault_injection/fault_injection_filter.h +1 -0
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h +3 -3
- data/src/core/ext/filters/http/client/http_client_filter.cc +1 -0
- data/src/core/ext/filters/http/client/http_client_filter.h +1 -0
- data/src/core/ext/filters/http/client_authority_filter.cc +1 -0
- data/src/core/ext/filters/http/client_authority_filter.h +1 -0
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +10 -15
- data/src/core/ext/filters/http/message_compress/compression_filter.h +2 -0
- data/src/core/ext/filters/http/server/http_server_filter.cc +2 -2
- data/src/core/ext/filters/http/server/http_server_filter.h +1 -0
- data/src/core/ext/filters/message_size/message_size_filter.cc +6 -9
- data/src/core/ext/filters/message_size/message_size_filter.h +6 -6
- data/src/core/ext/filters/rbac/rbac_filter.cc +2 -5
- data/src/core/ext/filters/rbac/rbac_filter.h +1 -0
- data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +2 -2
- data/src/core/ext/filters/rbac/rbac_service_config_parser.h +1 -1
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +2 -6
- data/src/core/ext/filters/stateful_session/stateful_session_filter.h +1 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +3 -3
- data/src/core/ext/transport/chttp2/alpn/alpn.cc +1 -1
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +8 -25
- data/src/core/ext/transport/chttp2/client/chttp2_connector.h +0 -5
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +223 -148
- data/src/core/ext/transport/chttp2/server/chttp2_server.h +33 -0
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +131 -107
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +8 -8
- data/src/core/ext/transport/chttp2/transport/context_list_entry.h +1 -1
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +2 -4
- data/src/core/ext/transport/chttp2/transport/flow_control.h +0 -2
- data/src/core/ext/transport/chttp2/transport/frame_ping.cc +4 -6
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +1 -2
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +9 -5
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +5 -4
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +9 -2
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +5 -7
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +1 -1
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +3 -8
- data/src/core/ext/transport/chttp2/transport/http2_settings.cc +1 -1
- data/src/core/ext/transport/chttp2/transport/http2_settings.h +1 -1
- data/src/core/ext/transport/chttp2/transport/internal.h +29 -19
- data/src/core/ext/transport/chttp2/transport/parsing.cc +15 -25
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.cc +0 -2
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.h +0 -2
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc +29 -13
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.h +5 -4
- data/src/core/ext/transport/chttp2/transport/stream_lists.cc +3 -5
- data/src/core/ext/transport/chttp2/transport/writing.cc +24 -25
- data/src/core/ext/transport/inproc/inproc_transport.cc +56 -32
- data/src/core/ext/transport/inproc/inproc_transport.h +1 -3
- data/src/core/ext/transport/inproc/legacy_inproc_transport.cc +13 -15
- data/src/core/ext/transport/inproc/legacy_inproc_transport.h +0 -2
- data/src/core/handshaker/handshaker.cc +6 -14
- data/src/core/handshaker/http_connect/http_connect_handshaker.cc +9 -17
- data/src/core/handshaker/http_connect/http_proxy_mapper.cc +3 -2
- data/src/core/handshaker/security/secure_endpoint.cc +38 -32
- data/src/core/handshaker/security/secure_endpoint.h +0 -2
- data/src/core/handshaker/security/security_handshaker.cc +25 -37
- data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc +2 -1
- data/src/core/lib/address_utils/parse_address.cc +27 -39
- data/src/core/lib/address_utils/sockaddr_utils.cc +5 -6
- data/src/core/lib/avl/avl.h +1 -1
- data/src/core/lib/channel/channel_args.cc +13 -17
- data/src/core/lib/channel/channel_args.h +19 -8
- data/src/core/lib/channel/channel_stack.cc +5 -63
- data/src/core/lib/channel/channel_stack.h +13 -37
- data/src/core/lib/channel/channel_stack_builder.h +0 -5
- data/src/core/lib/channel/channel_stack_builder_impl.cc +0 -142
- data/src/core/lib/channel/channel_stack_builder_impl.h +0 -2
- data/src/core/lib/channel/connected_channel.cc +37 -676
- data/src/core/lib/channel/promise_based_filter.cc +41 -47
- data/src/core/lib/channel/promise_based_filter.h +124 -477
- data/src/core/lib/channel/status_util.cc +1 -1
- data/src/core/lib/compression/compression.cc +1 -1
- data/src/core/lib/compression/message_compress.cc +6 -6
- data/src/core/lib/config/config_vars.cc +2 -7
- data/src/core/lib/config/config_vars.h +1 -5
- data/src/core/lib/debug/event_log.h +1 -1
- data/src/core/lib/debug/trace.cc +43 -59
- data/src/core/lib/debug/trace.h +2 -97
- data/src/core/lib/debug/trace_flags.cc +255 -0
- data/src/core/lib/debug/trace_flags.h +133 -0
- data/src/core/lib/debug/trace_impl.h +115 -0
- data/src/core/lib/event_engine/ares_resolver.cc +5 -7
- data/src/core/lib/event_engine/ares_resolver.h +1 -3
- data/src/core/lib/event_engine/cf_engine/cf_engine.cc +1 -1
- data/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc +17 -22
- data/src/core/lib/event_engine/event_engine.cc +29 -4
- data/src/core/lib/event_engine/extensions/supports_fd.h +7 -0
- data/src/core/lib/event_engine/extensions/tcp_trace.h +43 -0
- data/src/core/lib/event_engine/forkable.cc +4 -5
- data/src/core/lib/event_engine/forkable.h +0 -11
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +10 -11
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +4 -3
- data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc +1 -1
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +19 -33
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +3 -2
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +24 -7
- data/src/core/lib/event_engine/posix_engine/posix_engine.h +2 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +14 -16
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +18 -22
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +13 -17
- data/src/core/lib/event_engine/posix_engine/timer.cc +1 -1
- data/src/core/lib/event_engine/posix_engine/timer_manager.cc +4 -6
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +2 -1
- data/src/core/lib/event_engine/shim.cc +1 -1
- data/src/core/lib/event_engine/tcp_socket_utils.cc +6 -8
- data/src/core/lib/event_engine/thread_local.h +1 -1
- data/src/core/lib/event_engine/thread_pool/thread_count.h +1 -1
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc +19 -21
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h +3 -6
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc +14 -13
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h +4 -3
- data/src/core/lib/event_engine/trace.h +6 -17
- data/src/core/lib/event_engine/windows/iocp.h +1 -1
- data/src/core/lib/event_engine/windows/win_socket.cc +23 -17
- data/src/core/lib/event_engine/windows/win_socket.h +4 -5
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +6 -9
- data/src/core/lib/event_engine/windows/windows_engine.cc +201 -87
- data/src/core/lib/event_engine/windows/windows_engine.h +136 -25
- data/src/core/lib/event_engine/windows/windows_listener.cc +12 -23
- data/src/core/lib/experiments/experiments.cc +35 -151
- data/src/core/lib/experiments/experiments.h +12 -45
- data/src/core/lib/gprpp/bitset.h +1 -1
- data/src/core/lib/gprpp/crash.cc +2 -3
- data/src/core/lib/gprpp/dual_ref_counted.h +45 -33
- data/src/core/lib/gprpp/dump_args.cc +54 -0
- data/src/core/lib/gprpp/dump_args.h +69 -0
- data/src/core/lib/gprpp/glob.cc +70 -0
- data/src/core/lib/gprpp/glob.h +29 -0
- data/src/core/lib/gprpp/per_cpu.cc +1 -1
- data/src/core/lib/gprpp/posix/stat.cc +3 -4
- data/src/core/lib/gprpp/posix/thd.cc +8 -9
- data/src/core/lib/gprpp/ref_counted.h +30 -22
- data/src/core/lib/gprpp/single_set_ptr.h +5 -3
- data/src/core/lib/gprpp/status_helper.cc +11 -30
- data/src/core/lib/gprpp/status_helper.h +3 -31
- data/src/core/lib/gprpp/time.cc +3 -4
- data/src/core/lib/gprpp/time.h +3 -2
- data/src/core/lib/gprpp/unique_type_name.h +1 -1
- data/src/core/lib/gprpp/validation_errors.cc +10 -1
- data/src/core/lib/gprpp/validation_errors.h +11 -0
- data/src/core/lib/gprpp/windows/stat.cc +3 -4
- data/src/core/lib/gprpp/windows/thd.cc +3 -2
- data/src/core/lib/gprpp/work_serializer.cc +48 -57
- data/src/core/lib/iomgr/buffer_list.cc +4 -2
- data/src/core/lib/iomgr/call_combiner.cc +18 -27
- data/src/core/lib/iomgr/call_combiner.h +1 -3
- data/src/core/lib/iomgr/cfstream_handle.cc +4 -6
- data/src/core/lib/iomgr/closure.h +2 -4
- data/src/core/lib/iomgr/combiner.cc +6 -8
- data/src/core/lib/iomgr/combiner.h +0 -2
- data/src/core/lib/iomgr/endpoint.cc +0 -6
- data/src/core/lib/iomgr/endpoint.h +0 -2
- data/src/core/lib/iomgr/endpoint_cfstream.cc +19 -41
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +1 -1
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +5 -6
- data/src/core/lib/iomgr/error.cc +13 -21
- data/src/core/lib/iomgr/error.h +1 -1
- data/src/core/lib/iomgr/ev_apple.cc +3 -5
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +43 -42
- data/src/core/lib/iomgr/ev_poll_posix.cc +38 -29
- data/src/core/lib/iomgr/ev_posix.cc +8 -9
- data/src/core/lib/iomgr/ev_posix.h +10 -7
- data/src/core/lib/iomgr/event_engine_shims/closure.cc +2 -2
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +14 -28
- data/src/core/lib/iomgr/exec_ctx.cc +2 -2
- data/src/core/lib/iomgr/exec_ctx.h +1 -1
- data/src/core/lib/iomgr/executor.cc +6 -15
- data/src/core/lib/iomgr/executor.h +1 -1
- data/src/core/lib/iomgr/fork_posix.cc +8 -10
- data/src/core/lib/iomgr/fork_windows.cc +3 -1
- data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +2 -3
- data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +3 -5
- data/src/core/lib/iomgr/internal_errqueue.cc +4 -2
- data/src/core/lib/iomgr/iocp_windows.cc +4 -3
- data/src/core/lib/iomgr/iomgr.cc +13 -17
- data/src/core/lib/iomgr/lockfree_event.cc +3 -5
- data/src/core/lib/iomgr/pollset.h +0 -2
- data/src/core/lib/iomgr/pollset_windows.cc +0 -2
- data/src/core/lib/iomgr/resolve_address_posix.cc +7 -14
- data/src/core/lib/iomgr/resolve_address_windows.cc +1 -1
- data/src/core/lib/iomgr/socket_factory_posix.cc +1 -1
- data/src/core/lib/iomgr/socket_mutator.cc +1 -1
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +11 -17
- data/src/core/lib/iomgr/socket_windows.cc +4 -6
- data/src/core/lib/iomgr/tcp_client_cfstream.cc +3 -5
- data/src/core/lib/iomgr/tcp_client_posix.cc +9 -15
- data/src/core/lib/iomgr/tcp_client_windows.cc +2 -4
- data/src/core/lib/iomgr/tcp_posix.cc +57 -84
- data/src/core/lib/iomgr/tcp_posix.h +0 -2
- data/src/core/lib/iomgr/tcp_server_posix.cc +3 -3
- data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +4 -6
- data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +7 -7
- data/src/core/lib/iomgr/tcp_server_windows.cc +10 -16
- data/src/core/lib/iomgr/tcp_windows.cc +25 -41
- data/src/core/lib/iomgr/timer_generic.cc +17 -20
- data/src/core/lib/iomgr/timer_heap.cc +1 -1
- data/src/core/lib/iomgr/timer_manager.cc +17 -30
- data/src/core/lib/iomgr/unix_sockets_posix.cc +1 -1
- data/src/core/lib/iomgr/vsock.cc +1 -1
- data/src/core/lib/iomgr/wakeup_fd_pipe.cc +3 -3
- data/src/core/lib/promise/activity.h +27 -4
- data/src/core/lib/promise/cancel_callback.h +24 -0
- data/src/core/lib/promise/context.h +11 -0
- data/src/core/lib/promise/detail/basic_seq.h +1 -2
- data/src/core/lib/promise/detail/join_state.h +354 -398
- data/src/core/lib/promise/detail/promise_like.h +13 -6
- data/src/core/lib/promise/detail/seq_state.h +1178 -1178
- data/src/core/lib/promise/for_each.h +6 -6
- data/src/core/lib/promise/interceptor_list.h +6 -7
- data/src/core/lib/promise/latch.h +9 -9
- data/src/core/lib/promise/map.h +17 -0
- data/src/core/lib/promise/observable.h +182 -0
- data/src/core/lib/promise/party.cc +7 -8
- data/src/core/lib/promise/party.h +10 -8
- data/src/core/lib/promise/pipe.h +16 -35
- data/src/core/lib/promise/promise.h +1 -0
- data/src/core/lib/promise/status_flag.h +2 -0
- data/src/core/lib/resource_quota/arena.cc +56 -79
- data/src/core/lib/resource_quota/arena.h +118 -209
- data/src/core/lib/resource_quota/memory_quota.cc +12 -13
- data/src/core/lib/resource_quota/memory_quota.h +2 -3
- data/src/core/lib/resource_quota/periodic_update.cc +1 -1
- data/src/core/lib/resource_quota/resource_quota.h +1 -1
- data/src/core/lib/security/authorization/authorization_policy_provider.h +1 -1
- data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +1 -1
- data/src/core/lib/security/authorization/evaluate_args.cc +6 -8
- data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +5 -6
- data/src/core/lib/security/authorization/grpc_server_authz_filter.h +1 -0
- data/src/core/lib/security/authorization/matchers.cc +3 -3
- data/src/core/lib/security/certificate_provider/certificate_provider_factory.h +2 -2
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +2 -3
- data/src/core/lib/security/context/security_context.cc +12 -13
- data/src/core/lib/security/context/security_context.h +31 -8
- data/src/core/lib/security/credentials/alts/alts_credentials.h +1 -1
- data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +3 -1
- data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +2 -3
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +5 -5
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +3 -3
- data/src/core/lib/security/credentials/call_creds_util.cc +2 -1
- data/src/core/lib/security/credentials/channel_creds_registry.h +2 -2
- data/src/core/lib/security/credentials/channel_creds_registry_init.cc +5 -3
- data/src/core/lib/security/credentials/composite/composite_credentials.h +1 -1
- data/src/core/lib/security/credentials/credentials.cc +6 -6
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +4 -4
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +2 -2
- data/src/core/lib/security/credentials/external/external_account_credentials.cc +9 -11
- data/src/core/lib/security/credentials/external/external_account_credentials.h +3 -3
- data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +2 -2
- data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +4 -4
- data/src/core/lib/security/credentials/external/url_external_account_credentials.h +1 -1
- data/src/core/lib/security/credentials/fake/fake_credentials.h +1 -1
- data/src/core/lib/security/credentials/google_default/credentials_generic.cc +3 -3
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +11 -11
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +1 -1
- data/src/core/lib/security/credentials/iam/iam_credentials.h +1 -1
- data/src/core/lib/security/credentials/jwt/json_token.cc +14 -15
- data/src/core/lib/security/credentials/jwt/json_token.h +1 -1
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +6 -5
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +1 -1
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +50 -54
- data/src/core/lib/security/credentials/jwt/jwt_verifier.h +1 -1
- data/src/core/lib/security/credentials/local/local_credentials.h +1 -1
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +12 -11
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +4 -4
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +6 -7
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +1 -3
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +15 -21
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +21 -30
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +4 -4
- data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +3 -5
- data/src/core/lib/security/credentials/tls/tls_credentials.cc +14 -16
- data/src/core/lib/security/credentials/tls/tls_utils.cc +4 -4
- data/src/core/lib/security/credentials/xds/xds_credentials.cc +1 -1
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +13 -16
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +15 -12
- data/src/core/lib/security/security_connector/load_system_roots_supported.cc +6 -6
- data/src/core/lib/security/security_connector/load_system_roots_windows.cc +1 -1
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +8 -12
- data/src/core/lib/security/security_connector/security_connector.cc +1 -4
- data/src/core/lib/security/security_connector/security_connector.h +1 -3
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +17 -19
- data/src/core/lib/security/security_connector/ssl_utils.cc +19 -21
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +29 -40
- data/src/core/lib/security/transport/auth_filters.h +1 -0
- data/src/core/lib/security/transport/client_auth_filter.cc +7 -13
- data/src/core/lib/security/transport/server_auth_filter.cc +3 -8
- data/src/core/lib/security/util/json_util.h +1 -1
- data/src/core/lib/slice/slice.h +1 -1
- data/src/core/lib/slice/slice_refcount.h +2 -4
- data/src/core/lib/slice/slice_string_helpers.cc +1 -1
- data/src/core/lib/surface/api_trace.h +1 -3
- data/src/core/lib/surface/call.cc +64 -3739
- data/src/core/lib/surface/call.h +41 -143
- data/src/core/lib/surface/call_log_batch.cc +1 -1
- data/src/core/lib/surface/call_utils.cc +276 -0
- data/src/core/lib/surface/call_utils.h +449 -0
- data/src/core/lib/surface/channel.cc +8 -3
- data/src/core/lib/surface/channel.h +10 -7
- data/src/core/lib/surface/channel_create.cc +14 -6
- data/src/core/lib/surface/channel_create.h +3 -2
- data/src/core/lib/surface/channel_init.cc +21 -77
- data/src/core/lib/surface/channel_init.h +19 -97
- data/src/core/lib/surface/client_call.cc +419 -0
- data/src/core/lib/surface/client_call.h +180 -0
- data/src/core/lib/surface/completion_queue.cc +28 -33
- data/src/core/lib/surface/completion_queue.h +0 -8
- data/src/core/lib/surface/filter_stack_call.cc +1157 -0
- data/src/core/lib/surface/filter_stack_call.h +369 -0
- data/src/core/lib/surface/init.cc +7 -6
- data/src/core/lib/surface/lame_client.cc +1 -1
- data/src/core/lib/surface/legacy_channel.cc +43 -30
- data/src/core/lib/surface/legacy_channel.h +9 -18
- data/src/core/lib/surface/server_call.cc +222 -0
- data/src/core/lib/surface/server_call.h +167 -0
- data/src/core/lib/surface/version.cc +2 -2
- data/src/core/lib/transport/bdp_estimator.cc +3 -5
- data/src/core/lib/transport/bdp_estimator.h +2 -4
- data/src/core/lib/transport/call_arena_allocator.h +9 -7
- data/src/core/lib/transport/call_destination.h +76 -0
- data/src/core/lib/transport/call_filters.cc +28 -10
- data/src/core/lib/transport/call_filters.h +128 -22
- data/src/core/lib/transport/call_spine.cc +5 -6
- data/src/core/lib/transport/call_spine.h +159 -334
- data/src/core/lib/transport/connectivity_state.cc +8 -10
- data/src/core/lib/transport/connectivity_state.h +0 -2
- data/src/core/lib/transport/interception_chain.cc +155 -0
- data/src/core/lib/transport/interception_chain.h +236 -0
- data/src/core/lib/transport/metadata_batch.h +10 -1
- data/src/core/lib/transport/metadata_info.h +1 -1
- data/src/core/lib/transport/transport.cc +3 -6
- data/src/core/lib/transport/transport.h +43 -40
- data/src/core/load_balancing/child_policy_handler.cc +8 -8
- data/src/core/load_balancing/endpoint_list.cc +5 -5
- data/src/core/load_balancing/endpoint_list.h +1 -1
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.cc +1 -0
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.h +1 -0
- data/src/core/load_balancing/grpclb/grpclb.cc +25 -29
- data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc +1 -1
- data/src/core/load_balancing/grpclb/load_balancer_api.cc +3 -4
- data/src/core/load_balancing/health_check_client.cc +10 -13
- data/src/core/load_balancing/lb_policy.cc +5 -8
- data/src/core/load_balancing/lb_policy.h +19 -3
- data/src/core/load_balancing/lb_policy_factory.h +1 -1
- data/src/core/load_balancing/lb_policy_registry.cc +2 -3
- data/src/core/load_balancing/lb_policy_registry.h +1 -1
- data/src/core/load_balancing/oob_backend_metric.cc +2 -4
- data/src/core/load_balancing/outlier_detection/outlier_detection.cc +33 -35
- data/src/core/load_balancing/outlier_detection/outlier_detection.h +3 -3
- data/src/core/load_balancing/pick_first/pick_first.cc +65 -65
- data/src/core/load_balancing/priority/priority.cc +26 -28
- data/src/core/load_balancing/ring_hash/ring_hash.cc +11 -13
- data/src/core/load_balancing/ring_hash/ring_hash.h +3 -3
- data/src/core/load_balancing/rls/rls.cc +82 -82
- data/src/core/load_balancing/round_robin/round_robin.cc +17 -20
- data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +54 -43
- data/src/core/load_balancing/weighted_target/weighted_target.cc +21 -24
- data/src/core/load_balancing/xds/cds.cc +14 -16
- data/src/core/load_balancing/xds/xds_cluster_impl.cc +16 -18
- data/src/core/load_balancing/xds/xds_cluster_manager.cc +15 -17
- data/src/core/load_balancing/xds/xds_override_host.cc +40 -41
- data/src/core/load_balancing/xds/xds_override_host.h +3 -3
- data/src/core/load_balancing/xds/xds_wrr_locality.cc +10 -12
- data/src/core/plugin_registry/grpc_plugin_registry.cc +5 -1
- data/src/core/resolver/binder/binder_resolver.cc +3 -2
- data/src/core/resolver/dns/c_ares/dns_resolver_ares.cc +3 -2
- data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +1 -1
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc +7 -14
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.h +1 -5
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +1 -1
- data/src/core/resolver/dns/dns_resolver_plugin.cc +6 -5
- data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc +4 -9
- data/src/core/resolver/dns/event_engine/service_config_helper.cc +5 -5
- data/src/core/resolver/dns/native/dns_resolver.cc +8 -9
- data/src/core/resolver/endpoint_addresses.cc +1 -1
- data/src/core/resolver/fake/fake_resolver.cc +1 -1
- data/src/core/resolver/fake/fake_resolver.h +1 -1
- data/src/core/resolver/google_c2p/google_c2p_resolver.cc +13 -14
- data/src/core/resolver/polling_resolver.cc +30 -35
- data/src/core/resolver/resolver.cc +2 -6
- data/src/core/resolver/resolver.h +0 -2
- data/src/core/resolver/resolver_registry.cc +6 -8
- data/src/core/resolver/sockaddr/sockaddr_resolver.cc +3 -3
- data/src/core/resolver/xds/xds_dependency_manager.cc +22 -23
- data/src/core/resolver/xds/xds_resolver.cc +13 -15
- data/src/core/server/server.cc +269 -389
- data/src/core/server/server.h +37 -19
- data/src/core/server/server_call_tracer_filter.cc +7 -14
- data/src/core/server/server_config_selector.h +1 -1
- data/src/core/server/server_config_selector_filter.cc +3 -3
- data/src/core/server/server_interface.h +2 -0
- data/src/core/server/xds_channel_stack_modifier.cc +1 -1
- data/src/core/server/xds_channel_stack_modifier.h +1 -1
- data/src/core/server/xds_server_config_fetcher.cc +1 -4
- data/src/core/service_config/service_config.h +1 -1
- data/src/core/service_config/service_config_call_data.h +13 -11
- data/src/core/service_config/service_config_channel_arg_filter.cc +6 -4
- data/src/core/service_config/service_config_impl.cc +5 -5
- data/src/core/service_config/service_config_impl.h +1 -1
- data/src/core/service_config/service_config_parser.cc +3 -6
- data/src/core/service_config/service_config_parser.h +1 -1
- data/src/core/{lib/channel → telemetry}/call_tracer.cc +20 -30
- data/src/core/{lib/channel → telemetry}/call_tracer.h +32 -9
- data/src/core/{lib/debug → telemetry}/histogram_view.cc +1 -1
- data/src/core/{lib/debug → telemetry}/histogram_view.h +3 -3
- data/src/core/telemetry/metrics.cc +178 -0
- data/src/core/telemetry/metrics.h +562 -0
- data/src/core/{lib/debug → telemetry}/stats.cc +1 -1
- data/src/core/{lib/debug → telemetry}/stats.h +5 -5
- data/src/core/{lib/debug → telemetry}/stats_data.cc +1 -1
- data/src/core/{lib/debug → telemetry}/stats_data.h +4 -4
- data/src/core/{lib/channel → telemetry}/tcp_tracer.h +3 -3
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +12 -13
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +25 -27
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +32 -33
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +2 -1
- data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +2 -1
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +5 -4
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +5 -3
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +1 -1
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +4 -3
- data/src/core/tsi/fake_transport_security.cc +14 -17
- data/src/core/tsi/local_transport_security.cc +6 -5
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +4 -4
- data/src/core/tsi/ssl_transport_security.cc +76 -81
- data/src/core/tsi/ssl_transport_security_utils.cc +74 -18
- data/src/core/tsi/ssl_transport_security_utils.h +11 -0
- data/src/core/tsi/transport_security.cc +0 -4
- data/src/core/tsi/transport_security.h +0 -2
- data/src/core/tsi/transport_security_interface.h +0 -4
- data/src/core/{lib/gpr → util}/alloc.h +3 -3
- data/src/core/{lib/gpr → util}/android/log.cc +0 -19
- data/src/core/{lib/gpr → util}/atm.cc +1 -1
- data/src/core/{ext/gcp/metadata_query.cc → util/gcp_metadata_query.cc} +25 -26
- data/src/core/{ext/gcp/metadata_query.h → util/gcp_metadata_query.h} +11 -11
- data/src/core/{lib/http → util/http_client}/format_request.cc +4 -3
- data/src/core/{lib/http → util/http_client}/format_request.h +6 -5
- data/src/core/{lib/http → util/http_client}/httpcli.cc +9 -10
- data/src/core/{lib/http → util/http_client}/httpcli.h +6 -5
- data/src/core/{lib/http → util/http_client}/httpcli_security_connector.cc +9 -9
- data/src/core/{lib/http → util/http_client}/httpcli_ssl_credentials.h +5 -4
- data/src/core/{lib/http → util/http_client}/parser.cc +4 -5
- data/src/core/{lib/http → util/http_client}/parser.h +5 -6
- data/src/core/{lib → util}/json/json.h +5 -4
- data/src/core/{lib → util}/json/json_args.h +5 -5
- data/src/core/{lib → util}/json/json_channel_args.h +6 -6
- data/src/core/{lib → util}/json/json_object_loader.cc +3 -2
- data/src/core/{lib → util}/json/json_object_loader.h +7 -7
- data/src/core/{lib → util}/json/json_reader.cc +3 -2
- data/src/core/{lib → util}/json/json_reader.h +6 -6
- data/src/core/{lib → util}/json/json_util.cc +4 -4
- data/src/core/{lib → util}/json/json_util.h +6 -6
- data/src/core/{lib → util}/json/json_writer.cc +3 -3
- data/src/core/{lib → util}/json/json_writer.h +6 -6
- data/src/core/{lib/gpr → util}/linux/log.cc +0 -45
- data/src/core/{lib/gpr → util}/log.cc +45 -12
- data/src/core/{lib/gpr → util}/msys/tmpfile.cc +2 -2
- data/src/core/{lib/gpr → util}/posix/cpu.cc +1 -1
- data/src/core/{lib/gpr → util}/posix/log.cc +0 -42
- data/src/core/{lib/gpr → util}/posix/time.cc +1 -1
- data/src/core/{lib/gpr → util}/posix/tmpfile.cc +2 -2
- data/src/core/{lib/gpr → util}/spinlock.h +3 -3
- data/src/core/{lib/gpr → util}/string.cc +2 -2
- data/src/core/{lib/gpr → util}/string.h +3 -3
- data/src/core/{lib/gpr → util}/time_precise.cc +1 -1
- data/src/core/{lib/gpr → util}/time_precise.h +3 -3
- data/src/core/{lib/gpr → util}/tmpfile.h +3 -3
- data/src/core/{lib/gpr → util}/useful.h +3 -3
- data/src/core/{lib/gpr → util}/windows/log.cc +1 -44
- data/src/core/{lib/gpr → util}/windows/string.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/string_util.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/time.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/tmpfile.cc +1 -1
- data/src/core/xds/grpc/certificate_provider_store.cc +3 -3
- data/src/core/xds/grpc/certificate_provider_store.h +4 -4
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc +3 -3
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.h +3 -3
- data/src/core/xds/grpc/xds_audit_logger_registry.h +1 -1
- data/src/core/xds/grpc/xds_bootstrap_grpc.cc +5 -5
- data/src/core/xds/grpc/xds_bootstrap_grpc.h +3 -3
- data/src/core/xds/grpc/xds_certificate_provider.h +1 -1
- data/src/core/xds/grpc/xds_client_grpc.cc +27 -23
- data/src/core/xds/grpc/xds_client_grpc.h +2 -2
- data/src/core/xds/grpc/xds_cluster.cc +4 -5
- data/src/core/xds/grpc/xds_cluster.h +1 -1
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.cc +2 -2
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.h +1 -1
- data/src/core/xds/grpc/xds_common_types.cc +1 -1
- data/src/core/xds/grpc/xds_common_types.h +1 -1
- data/src/core/xds/grpc/xds_endpoint.cc +4 -5
- data/src/core/xds/grpc/xds_http_fault_filter.cc +2 -2
- data/src/core/xds/grpc/xds_http_filters.h +2 -2
- data/src/core/xds/grpc/xds_http_rbac_filter.cc +3 -3
- data/src/core/xds/grpc/xds_http_stateful_session_filter.cc +2 -2
- data/src/core/xds/grpc/xds_lb_policy_registry.h +1 -1
- data/src/core/xds/grpc/xds_listener.cc +4 -6
- data/src/core/xds/grpc/xds_route_config.cc +7 -8
- data/src/core/xds/grpc/xds_transport_grpc.cc +2 -2
- data/src/core/xds/grpc/xds_transport_grpc.h +1 -1
- data/src/core/xds/xds_client/xds_api.cc +5 -9
- data/src/core/xds/xds_client/xds_bootstrap.cc +1 -1
- data/src/core/xds/xds_client/xds_bootstrap.h +1 -1
- data/src/core/xds/xds_client/xds_client.cc +39 -45
- data/src/core/xds/xds_client/xds_client.h +0 -3
- data/src/core/xds/xds_client/xds_client_stats.cc +20 -18
- data/src/core/xds/xds_client/xds_client_stats.h +2 -2
- data/src/ruby/ext/grpc/rb_call.c +8 -1
- data/src/ruby/ext/grpc/rb_completion_queue.c +15 -32
- data/src/ruby/ext/grpc/rb_completion_queue.h +7 -1
- data/src/ruby/ext/grpc/rb_server.c +39 -22
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +4 -0
- data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +12 -12
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +66 -41
- data/third_party/boringssl-with-bazel/src/crypto/dilithium/dilithium.c +1497 -0
- data/third_party/boringssl-with-bazel/src/crypto/dilithium/internal.h +58 -0
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +10 -3
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +2 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +5 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +45 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +33 -23
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/fips_shared_support.c +3 -6
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +9 -4
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +6 -0
- data/third_party/boringssl-with-bazel/src/crypto/internal.h +7 -0
- data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +0 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509/v3_utl.c +49 -16
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +0 -10
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +7 -17
- data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +8 -6
- data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +3 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/experimental/dilithium.h +125 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +0 -23
- data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +1 -1
- data/third_party/boringssl-with-bazel/src/ssl/internal.h +4 -10
- metadata +95 -91
- data/src/core/ext/transport/chttp2/transport/http_trace.cc +0 -19
- data/src/core/ext/transport/chttp2/transport/http_trace.h +0 -24
- data/src/core/ext/transport/inproc/inproc_plugin.cc +0 -23
- data/src/core/handshaker/security/tsi_error.cc +0 -31
- data/src/core/handshaker/security/tsi_error.h +0 -30
- data/src/core/lib/channel/channel_stack_trace.cc +0 -19
- data/src/core/lib/channel/channel_stack_trace.h +0 -24
- data/src/core/lib/channel/context.h +0 -105
- data/src/core/lib/channel/metrics.cc +0 -334
- data/src/core/lib/channel/metrics.h +0 -365
- data/src/core/lib/event_engine/trace.cc +0 -25
- data/src/core/lib/iomgr/ev_windows.cc +0 -30
- data/src/core/lib/promise/trace.cc +0 -20
- data/src/core/lib/promise/trace.h +0 -24
- data/src/core/lib/resource_quota/trace.cc +0 -19
- data/src/core/lib/resource_quota/trace.h +0 -24
- data/src/core/lib/slice/slice_refcount.cc +0 -20
- data/src/core/lib/surface/api_trace.cc +0 -25
- data/src/core/lib/surface/call_trace.h +0 -24
- data/src/core/lib/surface/wait_for_cq_end_op.cc +0 -75
- data/src/core/lib/surface/wait_for_cq_end_op.h +0 -72
- data/src/core/lib/transport/batch_builder.cc +0 -172
- data/src/core/lib/transport/batch_builder.h +0 -474
- data/src/core/resolver/xds/xds_resolver_trace.cc +0 -25
- data/src/core/resolver/xds/xds_resolver_trace.h +0 -30
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -122
- /data/src/core/{lib/gpr → util}/alloc.cc +0 -0
- /data/src/core/{lib/gpr → util}/iphone/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/linux/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/posix/string.cc +0 -0
- /data/src/core/{lib/gpr → util}/posix/sync.cc +0 -0
- /data/src/core/{lib/gpr → util}/sync.cc +0 -0
- /data/src/core/{lib/gpr → util}/sync_abseil.cc +0 -0
- /data/src/core/{lib/gpr → util}/time.cc +0 -0
- /data/src/core/{lib/gpr → util}/windows/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/windows/sync.cc +0 -0
@@ -25,8 +25,10 @@
|
|
25
25
|
|
26
26
|
#include <algorithm>
|
27
27
|
#include <atomic>
|
28
|
+
#include <cstdint>
|
28
29
|
#include <memory>
|
29
30
|
#include <new>
|
31
|
+
#include <queue>
|
30
32
|
#include <string>
|
31
33
|
#include <type_traits>
|
32
34
|
#include <utility>
|
@@ -34,6 +36,7 @@
|
|
34
36
|
|
35
37
|
#include "absl/base/thread_annotations.h"
|
36
38
|
#include "absl/log/check.h"
|
39
|
+
#include "absl/log/log.h"
|
37
40
|
#include "absl/status/status.h"
|
38
41
|
#include "absl/strings/str_cat.h"
|
39
42
|
#include "absl/strings/str_format.h"
|
@@ -57,21 +60,15 @@
|
|
57
60
|
|
58
61
|
#include "src/core/channelz/channelz.h"
|
59
62
|
#include "src/core/lib/channel/call_finalization.h"
|
60
|
-
#include "src/core/lib/channel/call_tracer.h"
|
61
63
|
#include "src/core/lib/channel/channel_stack.h"
|
62
|
-
#include "src/core/lib/channel/context.h"
|
63
64
|
#include "src/core/lib/channel/status_util.h"
|
64
65
|
#include "src/core/lib/compression/compression_internal.h"
|
65
|
-
#include "src/core/lib/debug/stats.h"
|
66
|
-
#include "src/core/lib/debug/stats_data.h"
|
67
66
|
#include "src/core/lib/experiments/experiments.h"
|
68
|
-
#include "src/core/lib/gpr/alloc.h"
|
69
|
-
#include "src/core/lib/gpr/time_precise.h"
|
70
|
-
#include "src/core/lib/gpr/useful.h"
|
71
67
|
#include "src/core/lib/gprpp/bitset.h"
|
72
68
|
#include "src/core/lib/gprpp/cpp_impl_of.h"
|
73
69
|
#include "src/core/lib/gprpp/crash.h"
|
74
70
|
#include "src/core/lib/gprpp/debug_location.h"
|
71
|
+
#include "src/core/lib/gprpp/match.h"
|
75
72
|
#include "src/core/lib/gprpp/ref_counted.h"
|
76
73
|
#include "src/core/lib/gprpp/ref_counted_ptr.h"
|
77
74
|
#include "src/core/lib/gprpp/status_helper.h"
|
@@ -82,10 +79,10 @@
|
|
82
79
|
#include "src/core/lib/promise/activity.h"
|
83
80
|
#include "src/core/lib/promise/all_ok.h"
|
84
81
|
#include "src/core/lib/promise/arena_promise.h"
|
82
|
+
#include "src/core/lib/promise/cancel_callback.h"
|
85
83
|
#include "src/core/lib/promise/context.h"
|
86
84
|
#include "src/core/lib/promise/latch.h"
|
87
85
|
#include "src/core/lib/promise/map.h"
|
88
|
-
#include "src/core/lib/promise/party.h"
|
89
86
|
#include "src/core/lib/promise/pipe.h"
|
90
87
|
#include "src/core/lib/promise/poll.h"
|
91
88
|
#include "src/core/lib/promise/race.h"
|
@@ -100,17 +97,17 @@
|
|
100
97
|
#include "src/core/lib/surface/channel.h"
|
101
98
|
#include "src/core/lib/surface/completion_queue.h"
|
102
99
|
#include "src/core/lib/surface/validate_metadata.h"
|
103
|
-
#include "src/core/lib/surface/wait_for_cq_end_op.h"
|
104
|
-
#include "src/core/lib/transport/batch_builder.h"
|
105
100
|
#include "src/core/lib/transport/error_utils.h"
|
101
|
+
#include "src/core/lib/transport/metadata.h"
|
106
102
|
#include "src/core/lib/transport/metadata_batch.h"
|
107
103
|
#include "src/core/lib/transport/transport.h"
|
108
104
|
#include "src/core/server/server_interface.h"
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
105
|
+
#include "src/core/telemetry/call_tracer.h"
|
106
|
+
#include "src/core/telemetry/stats.h"
|
107
|
+
#include "src/core/telemetry/stats_data.h"
|
108
|
+
#include "src/core/util/alloc.h"
|
109
|
+
#include "src/core/util/time_precise.h"
|
110
|
+
#include "src/core/util/useful.h"
|
114
111
|
|
115
112
|
namespace grpc_core {
|
116
113
|
|
@@ -121,10 +118,19 @@ using GrpcClosure = Closure;
|
|
121
118
|
///////////////////////////////////////////////////////////////////////////////
|
122
119
|
// Call
|
123
120
|
|
121
|
+
Call::Call(bool is_client, Timestamp send_deadline, RefCountedPtr<Arena> arena,
|
122
|
+
grpc_event_engine::experimental::EventEngine* event_engine)
|
123
|
+
: arena_(std::move(arena)),
|
124
|
+
send_deadline_(send_deadline),
|
125
|
+
is_client_(is_client),
|
126
|
+
event_engine_(event_engine) {
|
127
|
+
arena_->SetContext<Call>(this);
|
128
|
+
}
|
129
|
+
|
124
130
|
Call::ParentCall* Call::GetOrCreateParentCall() {
|
125
131
|
ParentCall* p = parent_call_.load(std::memory_order_acquire);
|
126
132
|
if (p == nullptr) {
|
127
|
-
p =
|
133
|
+
p = arena()->New<ParentCall>();
|
128
134
|
ParentCall* expected = nullptr;
|
129
135
|
if (!parent_call_.compare_exchange_strong(expected, p,
|
130
136
|
std::memory_order_release,
|
@@ -160,8 +166,8 @@ absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
|
|
160
166
|
"Census tracing propagation requested without Census context "
|
161
167
|
"propagation");
|
162
168
|
}
|
163
|
-
|
164
|
-
|
169
|
+
arena()->SetContext<census_context>(
|
170
|
+
parent->arena()->GetContext<census_context>());
|
165
171
|
} else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
166
172
|
return absl::UnknownError(
|
167
173
|
"Census context propagation requested without Census tracing "
|
@@ -243,28 +249,6 @@ void Call::PropagateCancellationToChildren() {
|
|
243
249
|
}
|
244
250
|
}
|
245
251
|
|
246
|
-
char* Call::GetPeer() {
|
247
|
-
Slice peer_slice = GetPeerString();
|
248
|
-
if (!peer_slice.empty()) {
|
249
|
-
absl::string_view peer_string_view = peer_slice.as_string_view();
|
250
|
-
char* peer_string =
|
251
|
-
static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
|
252
|
-
memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
|
253
|
-
peer_string[peer_string_view.size()] = '\0';
|
254
|
-
return peer_string;
|
255
|
-
}
|
256
|
-
char* peer_string = grpc_channel_get_target(channel_->c_ptr());
|
257
|
-
if (peer_string != nullptr) return peer_string;
|
258
|
-
return gpr_strdup("unknown");
|
259
|
-
}
|
260
|
-
|
261
|
-
void Call::DeleteThis() {
|
262
|
-
RefCountedPtr<Channel> channel = std::move(channel_);
|
263
|
-
Arena* arena = arena_;
|
264
|
-
this->~Call();
|
265
|
-
channel->DestroyArena(arena);
|
266
|
-
}
|
267
|
-
|
268
252
|
void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
|
269
253
|
grpc_metadata_batch& md) {
|
270
254
|
// TODO(juanlishen): If the user has already specified a compression
|
@@ -279,7 +263,7 @@ void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
|
|
279
263
|
op.data.send_initial_metadata.maybe_compression_level.level;
|
280
264
|
level_set = true;
|
281
265
|
} else {
|
282
|
-
const grpc_compression_options copts =
|
266
|
+
const grpc_compression_options copts = compression_options();
|
283
267
|
if (copts.default_level.is_set) {
|
284
268
|
level_set = true;
|
285
269
|
effective_compression_level = copts.default_level.level;
|
@@ -305,26 +289,25 @@ void Call::ProcessIncomingInitialMetadata(grpc_metadata_batch& md) {
|
|
305
289
|
Slice* peer_string = md.get_pointer(PeerString());
|
306
290
|
if (peer_string != nullptr) SetPeerString(peer_string->Ref());
|
307
291
|
|
308
|
-
|
309
|
-
md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
292
|
+
SetIncomingCompressionAlgorithm(
|
293
|
+
md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE));
|
310
294
|
encodings_accepted_by_peer_ =
|
311
295
|
md.Take(GrpcAcceptEncodingMetadata())
|
312
296
|
.value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
|
313
297
|
|
314
|
-
const grpc_compression_options
|
315
|
-
channel_->compression_options();
|
298
|
+
const grpc_compression_options copts = compression_options();
|
316
299
|
const grpc_compression_algorithm compression_algorithm =
|
317
|
-
|
318
|
-
if (GPR_UNLIKELY(
|
319
|
-
|
320
|
-
|
300
|
+
incoming_compression_algorithm();
|
301
|
+
if (GPR_UNLIKELY(
|
302
|
+
!CompressionAlgorithmSet::FromUint32(copts.enabled_algorithms_bitset)
|
303
|
+
.IsSet(compression_algorithm))) {
|
321
304
|
// check if algorithm is supported by current channel config
|
322
305
|
HandleCompressionAlgorithmDisabled(compression_algorithm);
|
323
306
|
}
|
324
307
|
// GRPC_COMPRESS_NONE is always set.
|
325
308
|
DCHECK(encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
326
309
|
if (GPR_UNLIKELY(!encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
327
|
-
if (GRPC_TRACE_FLAG_ENABLED(
|
310
|
+
if (GRPC_TRACE_FLAG_ENABLED(compression)) {
|
328
311
|
HandleCompressionAlgorithmNotAccepted(compression_algorithm);
|
329
312
|
}
|
330
313
|
}
|
@@ -347,7 +330,7 @@ void Call::HandleCompressionAlgorithmDisabled(
|
|
347
330
|
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
348
331
|
std::string error_msg =
|
349
332
|
absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
|
350
|
-
|
333
|
+
LOG(ERROR) << error_msg;
|
351
334
|
CancelWithError(grpc_error_set_int(absl::UnimplementedError(error_msg),
|
352
335
|
StatusIntProperty::kRpcStatus,
|
353
336
|
GRPC_STATUS_UNIMPLEMENTED));
|
@@ -355,7 +338,7 @@ void Call::HandleCompressionAlgorithmDisabled(
|
|
355
338
|
|
356
339
|
void Call::UpdateDeadline(Timestamp deadline) {
|
357
340
|
ReleasableMutexLock lock(&deadline_mu_);
|
358
|
-
if (
|
341
|
+
if (GRPC_TRACE_FLAG_ENABLED(call)) {
|
359
342
|
gpr_log(GPR_DEBUG, "[call %p] UpdateDeadline from=%s to=%s", this,
|
360
343
|
deadline_.ToString().c_str(), deadline.ToString().c_str());
|
361
344
|
}
|
@@ -367,22 +350,20 @@ void Call::UpdateDeadline(Timestamp deadline) {
|
|
367
350
|
StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
|
368
351
|
return;
|
369
352
|
}
|
370
|
-
auto* const event_engine = channel()->event_engine();
|
371
353
|
if (deadline_ != Timestamp::InfFuture()) {
|
372
|
-
if (!
|
354
|
+
if (!event_engine_->Cancel(deadline_task_)) return;
|
373
355
|
} else {
|
374
356
|
InternalRef("deadline");
|
375
357
|
}
|
376
358
|
deadline_ = deadline;
|
377
|
-
deadline_task_ =
|
359
|
+
deadline_task_ = event_engine_->RunAfter(deadline - Timestamp::Now(), this);
|
378
360
|
}
|
379
361
|
|
380
362
|
void Call::ResetDeadline() {
|
381
363
|
{
|
382
364
|
MutexLock lock(&deadline_mu_);
|
383
365
|
if (deadline_ == Timestamp::InfFuture()) return;
|
384
|
-
|
385
|
-
if (!event_engine->Cancel(deadline_task_)) return;
|
366
|
+
if (!event_engine_->Cancel(deadline_task_)) return;
|
386
367
|
deadline_ = Timestamp::InfFuture();
|
387
368
|
}
|
388
369
|
InternalUnref("deadline[reset]");
|
@@ -391,3701 +372,41 @@ void Call::ResetDeadline() {
|
|
391
372
|
void Call::Run() {
|
392
373
|
ApplicationCallbackExecCtx callback_exec_ctx;
|
393
374
|
ExecCtx exec_ctx;
|
375
|
+
GRPC_TRACE_LOG(call, INFO)
|
376
|
+
<< "call deadline expired "
|
377
|
+
<< GRPC_DUMP_ARGS(Timestamp::Now(), send_deadline_);
|
394
378
|
CancelWithError(grpc_error_set_int(
|
395
379
|
absl::DeadlineExceededError("Deadline Exceeded"),
|
396
380
|
StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
|
397
381
|
InternalUnref("deadline[run]");
|
398
382
|
}
|
399
383
|
|
400
|
-
|
401
|
-
// FilterStackCall
|
402
|
-
// To be removed once promise conversion is complete
|
403
|
-
|
404
|
-
class FilterStackCall final : public Call {
|
405
|
-
public:
|
406
|
-
~FilterStackCall() override {
|
407
|
-
for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
|
408
|
-
if (context_[i].destroy) {
|
409
|
-
context_[i].destroy(context_[i].value);
|
410
|
-
}
|
411
|
-
}
|
412
|
-
gpr_free(static_cast<void*>(const_cast<char*>(final_info_.error_string)));
|
413
|
-
}
|
414
|
-
|
415
|
-
bool Completed() override {
|
416
|
-
return gpr_atm_acq_load(&received_final_op_atm_) != 0;
|
417
|
-
}
|
418
|
-
|
419
|
-
// TODO(ctiller): return absl::StatusOr<SomeSmartPointer<Call>>?
|
420
|
-
static grpc_error_handle Create(grpc_call_create_args* args,
|
421
|
-
grpc_call** out_call);
|
422
|
-
|
423
|
-
static Call* FromTopElem(grpc_call_element* elem) {
|
424
|
-
return FromCallStack(grpc_call_stack_from_top_element(elem));
|
425
|
-
}
|
426
|
-
|
427
|
-
grpc_call_stack* call_stack() override {
|
428
|
-
return reinterpret_cast<grpc_call_stack*>(
|
429
|
-
reinterpret_cast<char*>(this) +
|
430
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(*this)));
|
431
|
-
}
|
432
|
-
|
433
|
-
grpc_event_engine::experimental::EventEngine* event_engine() const override {
|
434
|
-
return channel()->event_engine();
|
435
|
-
}
|
436
|
-
|
437
|
-
grpc_call_element* call_elem(size_t idx) {
|
438
|
-
return grpc_call_stack_element(call_stack(), idx);
|
439
|
-
}
|
440
|
-
|
441
|
-
CallCombiner* call_combiner() { return &call_combiner_; }
|
442
|
-
|
443
|
-
void CancelWithError(grpc_error_handle error) override;
|
444
|
-
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
445
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
446
|
-
bool is_notify_tag_closure) override;
|
447
|
-
void ExternalRef() override { ext_ref_.Ref(); }
|
448
|
-
void ExternalUnref() override;
|
449
|
-
void InternalRef(const char* reason) override {
|
450
|
-
GRPC_CALL_STACK_REF(call_stack(), reason);
|
451
|
-
}
|
452
|
-
void InternalUnref(const char* reason) override {
|
453
|
-
GRPC_CALL_STACK_UNREF(call_stack(), reason);
|
454
|
-
}
|
455
|
-
|
456
|
-
void ContextSet(grpc_context_index elem, void* value,
|
457
|
-
void (*destroy)(void* value)) override;
|
458
|
-
void* ContextGet(grpc_context_index elem) const override {
|
459
|
-
return context_[elem].value;
|
460
|
-
}
|
461
|
-
|
462
|
-
bool is_trailers_only() const override {
|
463
|
-
bool result = is_trailers_only_;
|
464
|
-
DCHECK(!result || recv_initial_metadata_.TransportSize() == 0);
|
465
|
-
return result;
|
466
|
-
}
|
467
|
-
|
468
|
-
bool failed_before_recv_message() const override {
|
469
|
-
return call_failed_before_recv_message_;
|
470
|
-
}
|
471
|
-
|
472
|
-
absl::string_view GetServerAuthority() const override {
|
473
|
-
const Slice* authority_metadata =
|
474
|
-
recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
|
475
|
-
if (authority_metadata == nullptr) return "";
|
476
|
-
return authority_metadata->as_string_view();
|
477
|
-
}
|
478
|
-
|
479
|
-
static size_t InitialSizeEstimate() {
|
480
|
-
return sizeof(FilterStackCall) +
|
481
|
-
sizeof(BatchControl) * kMaxConcurrentBatches;
|
482
|
-
}
|
483
|
-
|
484
|
-
private:
|
485
|
-
class ScopedContext : public promise_detail::Context<Arena> {
|
486
|
-
public:
|
487
|
-
explicit ScopedContext(FilterStackCall* call)
|
488
|
-
: promise_detail::Context<Arena>(call->arena()) {}
|
489
|
-
};
|
490
|
-
|
491
|
-
static constexpr gpr_atm kRecvNone = 0;
|
492
|
-
static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
|
493
|
-
|
494
|
-
enum class PendingOp {
|
495
|
-
kRecvMessage,
|
496
|
-
kRecvInitialMetadata,
|
497
|
-
kRecvTrailingMetadata,
|
498
|
-
kSends
|
499
|
-
};
|
500
|
-
static intptr_t PendingOpMask(PendingOp op) {
|
501
|
-
return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
|
502
|
-
}
|
503
|
-
static std::string PendingOpString(intptr_t pending_ops) {
|
504
|
-
std::vector<absl::string_view> pending_op_strings;
|
505
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
|
506
|
-
pending_op_strings.push_back("kRecvMessage");
|
507
|
-
}
|
508
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
|
509
|
-
pending_op_strings.push_back("kRecvInitialMetadata");
|
510
|
-
}
|
511
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
|
512
|
-
pending_op_strings.push_back("kRecvTrailingMetadata");
|
513
|
-
}
|
514
|
-
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
515
|
-
pending_op_strings.push_back("kSends");
|
516
|
-
}
|
517
|
-
return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
|
518
|
-
}
|
519
|
-
struct BatchControl {
|
520
|
-
FilterStackCall* call_ = nullptr;
|
521
|
-
CallTracerAnnotationInterface* call_tracer_ = nullptr;
|
522
|
-
grpc_transport_stream_op_batch op_;
|
523
|
-
// Share memory for cq_completion and notify_tag as they are never needed
|
524
|
-
// simultaneously. Each byte used in this data structure count as six bytes
|
525
|
-
// per call, so any savings we can make are worthwhile,
|
526
|
-
|
527
|
-
// We use notify_tag to determine whether or not to send notification to the
|
528
|
-
// completion queue. Once we've made that determination, we can reuse the
|
529
|
-
// memory for cq_completion.
|
530
|
-
union {
|
531
|
-
grpc_cq_completion cq_completion;
|
532
|
-
struct {
|
533
|
-
// Any given op indicates completion by either (a) calling a closure or
|
534
|
-
// (b) sending a notification on the call's completion queue. If
|
535
|
-
// \a is_closure is true, \a tag indicates a closure to be invoked;
|
536
|
-
// otherwise, \a tag indicates the tag to be used in the notification to
|
537
|
-
// be sent to the completion queue.
|
538
|
-
void* tag;
|
539
|
-
bool is_closure;
|
540
|
-
} notify_tag;
|
541
|
-
} completion_data_;
|
542
|
-
grpc_closure start_batch_;
|
543
|
-
grpc_closure finish_batch_;
|
544
|
-
std::atomic<intptr_t> ops_pending_{0};
|
545
|
-
AtomicError batch_error_;
|
546
|
-
void set_pending_ops(uintptr_t ops) {
|
547
|
-
ops_pending_.store(ops, std::memory_order_release);
|
548
|
-
}
|
549
|
-
bool completed_batch_step(PendingOp op) {
|
550
|
-
auto mask = PendingOpMask(op);
|
551
|
-
auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
|
552
|
-
if (grpc_call_trace.enabled()) {
|
553
|
-
gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
|
554
|
-
PendingOpString(mask).c_str(),
|
555
|
-
PendingOpString(r & ~mask).c_str(),
|
556
|
-
completion_data_.notify_tag.tag);
|
557
|
-
}
|
558
|
-
CHECK_NE((r & mask), 0);
|
559
|
-
return r == mask;
|
560
|
-
}
|
561
|
-
|
562
|
-
void PostCompletion();
|
563
|
-
void FinishStep(PendingOp op);
|
564
|
-
void ProcessDataAfterMetadata();
|
565
|
-
void ReceivingStreamReady(grpc_error_handle error);
|
566
|
-
void ReceivingInitialMetadataReady(grpc_error_handle error);
|
567
|
-
void ReceivingTrailingMetadataReady(grpc_error_handle error);
|
568
|
-
void FinishBatch(grpc_error_handle error);
|
569
|
-
};
|
570
|
-
|
571
|
-
FilterStackCall(Arena* arena, const grpc_call_create_args& args)
|
572
|
-
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
573
|
-
args.channel->Ref()),
|
574
|
-
cq_(args.cq),
|
575
|
-
stream_op_payload_(context_) {
|
576
|
-
context_[GRPC_CONTEXT_CALL].value = this;
|
577
|
-
}
|
578
|
-
|
579
|
-
static void ReleaseCall(void* call, grpc_error_handle);
|
580
|
-
static void DestroyCall(void* call, grpc_error_handle);
|
581
|
-
|
582
|
-
static FilterStackCall* FromCallStack(grpc_call_stack* call_stack) {
|
583
|
-
return reinterpret_cast<FilterStackCall*>(
|
584
|
-
reinterpret_cast<char*>(call_stack) -
|
585
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)));
|
586
|
-
}
|
587
|
-
|
588
|
-
void ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
589
|
-
grpc_closure* start_batch_closure);
|
590
|
-
void SetFinalStatus(grpc_error_handle error);
|
591
|
-
BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
|
592
|
-
bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
|
593
|
-
bool is_trailing);
|
594
|
-
void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
|
595
|
-
void RecvInitialFilter(grpc_metadata_batch* b);
|
596
|
-
void RecvTrailingFilter(grpc_metadata_batch* b,
|
597
|
-
grpc_error_handle batch_error);
|
598
|
-
|
599
|
-
RefCount ext_ref_;
|
600
|
-
CallCombiner call_combiner_;
|
601
|
-
grpc_completion_queue* cq_;
|
602
|
-
grpc_polling_entity pollent_;
|
603
|
-
|
604
|
-
/// has grpc_call_unref been called
|
605
|
-
bool destroy_called_ = false;
|
606
|
-
// Trailers-only response status
|
607
|
-
bool is_trailers_only_ = false;
|
608
|
-
/// which ops are in-flight
|
609
|
-
bool sent_initial_metadata_ = false;
|
610
|
-
bool sending_message_ = false;
|
611
|
-
bool sent_final_op_ = false;
|
612
|
-
bool received_initial_metadata_ = false;
|
613
|
-
bool receiving_message_ = false;
|
614
|
-
bool requested_final_op_ = false;
|
615
|
-
gpr_atm received_final_op_atm_ = 0;
|
616
|
-
|
617
|
-
BatchControl* active_batches_[kMaxConcurrentBatches] = {};
|
618
|
-
grpc_transport_stream_op_batch_payload stream_op_payload_;
|
619
|
-
|
620
|
-
// first idx: is_receiving, second idx: is_trailing
|
621
|
-
grpc_metadata_batch send_initial_metadata_;
|
622
|
-
grpc_metadata_batch send_trailing_metadata_;
|
623
|
-
grpc_metadata_batch recv_initial_metadata_;
|
624
|
-
grpc_metadata_batch recv_trailing_metadata_;
|
625
|
-
|
626
|
-
// Buffered read metadata waiting to be returned to the application.
|
627
|
-
// Element 0 is initial metadata, element 1 is trailing metadata.
|
628
|
-
grpc_metadata_array* buffered_metadata_[2] = {};
|
629
|
-
|
630
|
-
// Call data useful used for reporting. Only valid after the call has
|
631
|
-
// completed
|
632
|
-
grpc_call_final_info final_info_;
|
633
|
-
|
634
|
-
// Contexts for various subsystems (security, tracing, ...).
|
635
|
-
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
636
|
-
|
637
|
-
SliceBuffer send_slice_buffer_;
|
638
|
-
absl::optional<SliceBuffer> receiving_slice_buffer_;
|
639
|
-
uint32_t receiving_stream_flags_;
|
640
|
-
|
641
|
-
bool call_failed_before_recv_message_ = false;
|
642
|
-
grpc_byte_buffer** receiving_buffer_ = nullptr;
|
643
|
-
grpc_slice receiving_slice_ = grpc_empty_slice();
|
644
|
-
grpc_closure receiving_stream_ready_;
|
645
|
-
grpc_closure receiving_initial_metadata_ready_;
|
646
|
-
grpc_closure receiving_trailing_metadata_ready_;
|
647
|
-
// Status about operation of call
|
648
|
-
bool sent_server_trailing_metadata_ = false;
|
649
|
-
gpr_atm cancelled_with_error_ = 0;
|
650
|
-
|
651
|
-
grpc_closure release_call_;
|
652
|
-
|
653
|
-
union {
|
654
|
-
struct {
|
655
|
-
grpc_status_code* status;
|
656
|
-
grpc_slice* status_details;
|
657
|
-
const char** error_string;
|
658
|
-
} client;
|
659
|
-
struct {
|
660
|
-
int* cancelled;
|
661
|
-
// backpointer to owning server if this is a server side call.
|
662
|
-
ServerInterface* core_server;
|
663
|
-
} server;
|
664
|
-
} final_op_;
|
665
|
-
AtomicError status_error_;
|
666
|
-
|
667
|
-
// recv_state can contain one of the following values:
|
668
|
-
// RECV_NONE : : no initial metadata and messages received
|
669
|
-
// RECV_INITIAL_METADATA_FIRST : received initial metadata first
|
670
|
-
// a batch_control* : received messages first
|
671
|
-
|
672
|
-
// +------1------RECV_NONE------3-----+
|
673
|
-
// | |
|
674
|
-
// | |
|
675
|
-
// v v
|
676
|
-
// RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
|
677
|
-
// | ^ | ^
|
678
|
-
// | | | |
|
679
|
-
// +-----2-----+ +-----4-----+
|
680
|
-
|
681
|
-
// For 1, 4: See receiving_initial_metadata_ready() function
|
682
|
-
// For 2, 3: See receiving_stream_ready() function
|
683
|
-
gpr_atm recv_state_ = 0;
|
684
|
-
};
|
685
|
-
|
686
|
-
grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
687
|
-
grpc_call** out_call) {
|
688
|
-
Channel* channel = args->channel.get();
|
689
|
-
|
690
|
-
auto add_init_error = [](grpc_error_handle* composite,
|
691
|
-
grpc_error_handle new_err) {
|
692
|
-
if (new_err.ok()) return;
|
693
|
-
if (composite->ok()) {
|
694
|
-
*composite = GRPC_ERROR_CREATE("Call creation failed");
|
695
|
-
}
|
696
|
-
*composite = grpc_error_add_child(*composite, new_err);
|
697
|
-
};
|
698
|
-
|
699
|
-
FilterStackCall* call;
|
700
|
-
grpc_error_handle error;
|
701
|
-
grpc_channel_stack* channel_stack = channel->channel_stack();
|
702
|
-
size_t call_alloc_size =
|
703
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
|
704
|
-
channel_stack->call_stack_size;
|
705
|
-
|
706
|
-
Arena* arena = channel->CreateArena();
|
707
|
-
call = new (arena->Alloc(call_alloc_size)) FilterStackCall(arena, *args);
|
708
|
-
DCHECK(FromC(call->c_ptr()) == call);
|
709
|
-
DCHECK(FromCallStack(call->call_stack()) == call);
|
710
|
-
*out_call = call->c_ptr();
|
711
|
-
grpc_slice path = grpc_empty_slice();
|
712
|
-
ScopedContext ctx(call);
|
713
|
-
if (call->is_client()) {
|
714
|
-
call->final_op_.client.status_details = nullptr;
|
715
|
-
call->final_op_.client.status = nullptr;
|
716
|
-
call->final_op_.client.error_string = nullptr;
|
717
|
-
global_stats().IncrementClientCallsCreated();
|
718
|
-
path = CSliceRef(args->path->c_slice());
|
719
|
-
call->send_initial_metadata_.Set(HttpPathMetadata(),
|
720
|
-
std::move(*args->path));
|
721
|
-
if (args->authority.has_value()) {
|
722
|
-
call->send_initial_metadata_.Set(HttpAuthorityMetadata(),
|
723
|
-
std::move(*args->authority));
|
724
|
-
}
|
725
|
-
call->send_initial_metadata_.Set(
|
726
|
-
GrpcRegisteredMethod(), reinterpret_cast<void*>(static_cast<uintptr_t>(
|
727
|
-
args->registered_method)));
|
728
|
-
channel_stack->stats_plugin_group->AddClientCallTracers(
|
729
|
-
Slice(CSliceRef(path)), args->registered_method, call->context_);
|
730
|
-
} else {
|
731
|
-
global_stats().IncrementServerCallsCreated();
|
732
|
-
call->final_op_.server.cancelled = nullptr;
|
733
|
-
call->final_op_.server.core_server = args->server;
|
734
|
-
// TODO(yashykt): In the future, we want to also enable stats and trace
|
735
|
-
// collecting from when the call is created at the transport. The idea is
|
736
|
-
// that the transport would create the call tracer and pass it in as part of
|
737
|
-
// the metadata.
|
738
|
-
// TODO(yijiem): OpenCensus and internal Census is still using this way to
|
739
|
-
// set server call tracer. We need to refactor them to stats plugins
|
740
|
-
// (including removing the client channel filters).
|
741
|
-
if (args->server != nullptr &&
|
742
|
-
args->server->server_call_tracer_factory() != nullptr) {
|
743
|
-
auto* server_call_tracer =
|
744
|
-
args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
|
745
|
-
arena, args->server->channel_args());
|
746
|
-
if (server_call_tracer != nullptr) {
|
747
|
-
// Note that we are setting both
|
748
|
-
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
749
|
-
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
750
|
-
// promise-based world, we would just a single tracer object for each
|
751
|
-
// stack (call, subchannel_call, server_call.)
|
752
|
-
call->ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
753
|
-
server_call_tracer, nullptr);
|
754
|
-
call->ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
755
|
-
}
|
756
|
-
}
|
757
|
-
channel_stack->stats_plugin_group->AddServerCallTracers(call->context_);
|
758
|
-
}
|
759
|
-
|
760
|
-
Call* parent = Call::FromC(args->parent);
|
761
|
-
if (parent != nullptr) {
|
762
|
-
add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
|
763
|
-
parent, args->propagation_mask)));
|
764
|
-
}
|
765
|
-
// initial refcount dropped by grpc_call_unref
|
766
|
-
grpc_call_element_args call_args = {
|
767
|
-
call->call_stack(), args->server_transport_data,
|
768
|
-
call->context_, path,
|
769
|
-
call->start_time(), call->send_deadline(),
|
770
|
-
call->arena(), &call->call_combiner_};
|
771
|
-
add_init_error(&error, grpc_call_stack_init(channel_stack, 1, DestroyCall,
|
772
|
-
call, &call_args));
|
773
|
-
// Publish this call to parent only after the call stack has been initialized.
|
774
|
-
if (parent != nullptr) {
|
775
|
-
call->PublishToParent(parent);
|
776
|
-
}
|
777
|
-
|
778
|
-
if (!error.ok()) {
|
779
|
-
call->CancelWithError(error);
|
780
|
-
}
|
781
|
-
if (args->cq != nullptr) {
|
782
|
-
CHECK(args->pollset_set_alternative == nullptr)
|
783
|
-
<< "Only one of 'cq' and 'pollset_set_alternative' should be "
|
784
|
-
"non-nullptr.";
|
785
|
-
GRPC_CQ_INTERNAL_REF(args->cq, "bind");
|
786
|
-
call->pollent_ =
|
787
|
-
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args->cq));
|
788
|
-
}
|
789
|
-
if (args->pollset_set_alternative != nullptr) {
|
790
|
-
call->pollent_ = grpc_polling_entity_create_from_pollset_set(
|
791
|
-
args->pollset_set_alternative);
|
792
|
-
}
|
793
|
-
if (!grpc_polling_entity_is_empty(&call->pollent_)) {
|
794
|
-
grpc_call_stack_set_pollset_or_pollset_set(call->call_stack(),
|
795
|
-
&call->pollent_);
|
796
|
-
}
|
797
|
-
|
798
|
-
if (call->is_client()) {
|
799
|
-
channelz::ChannelNode* channelz_channel = channel->channelz_node();
|
800
|
-
if (channelz_channel != nullptr) {
|
801
|
-
channelz_channel->RecordCallStarted();
|
802
|
-
}
|
803
|
-
} else if (call->final_op_.server.core_server != nullptr) {
|
804
|
-
channelz::ServerNode* channelz_node =
|
805
|
-
call->final_op_.server.core_server->channelz_node();
|
806
|
-
if (channelz_node != nullptr) {
|
807
|
-
channelz_node->RecordCallStarted();
|
808
|
-
}
|
809
|
-
}
|
810
|
-
|
811
|
-
if (args->send_deadline != Timestamp::InfFuture()) {
|
812
|
-
call->UpdateDeadline(args->send_deadline);
|
813
|
-
}
|
814
|
-
|
815
|
-
CSliceUnref(path);
|
816
|
-
|
817
|
-
return error;
|
818
|
-
}
|
819
|
-
|
820
|
-
void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
821
|
-
CHECK(cq);
|
822
|
-
|
823
|
-
if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
|
824
|
-
Crash("A pollset_set is already registered for this call.");
|
825
|
-
}
|
826
|
-
cq_ = cq;
|
827
|
-
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
828
|
-
pollent_ = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
|
829
|
-
grpc_call_stack_set_pollset_or_pollset_set(call_stack(), &pollent_);
|
830
|
-
}
|
831
|
-
|
832
|
-
void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
|
833
|
-
static_cast<FilterStackCall*>(call)->DeleteThis();
|
834
|
-
}
|
835
|
-
|
836
|
-
void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
837
|
-
auto* c = static_cast<FilterStackCall*>(call);
|
838
|
-
c->recv_initial_metadata_.Clear();
|
839
|
-
c->recv_trailing_metadata_.Clear();
|
840
|
-
c->receiving_slice_buffer_.reset();
|
841
|
-
ParentCall* pc = c->parent_call();
|
842
|
-
if (pc != nullptr) {
|
843
|
-
pc->~ParentCall();
|
844
|
-
}
|
845
|
-
if (c->cq_) {
|
846
|
-
GRPC_CQ_INTERNAL_UNREF(c->cq_, "bind");
|
847
|
-
}
|
848
|
-
|
849
|
-
grpc_error_handle status_error = c->status_error_.get();
|
850
|
-
grpc_error_get_status(status_error, c->send_deadline(),
|
851
|
-
&c->final_info_.final_status, nullptr, nullptr,
|
852
|
-
&(c->final_info_.error_string));
|
853
|
-
c->status_error_.set(absl::OkStatus());
|
854
|
-
c->final_info_.stats.latency =
|
855
|
-
gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time());
|
856
|
-
grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
|
857
|
-
GRPC_CLOSURE_INIT(&c->release_call_, ReleaseCall, c,
|
858
|
-
grpc_schedule_on_exec_ctx));
|
859
|
-
}
|
860
|
-
|
861
|
-
void FilterStackCall::ExternalUnref() {
|
862
|
-
if (GPR_LIKELY(!ext_ref_.Unref())) return;
|
863
|
-
|
864
|
-
ApplicationCallbackExecCtx callback_exec_ctx;
|
865
|
-
ExecCtx exec_ctx;
|
866
|
-
|
867
|
-
GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (this));
|
868
|
-
|
869
|
-
MaybeUnpublishFromParent();
|
870
|
-
|
871
|
-
CHECK(!destroy_called_);
|
872
|
-
destroy_called_ = true;
|
873
|
-
bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
|
874
|
-
if (cancel) {
|
875
|
-
CancelWithError(absl::CancelledError());
|
876
|
-
} else {
|
877
|
-
// Unset the call combiner cancellation closure. This has the
|
878
|
-
// effect of scheduling the previously set cancellation closure, if
|
879
|
-
// any, so that it can release any internal references it may be
|
880
|
-
// holding to the call stack.
|
881
|
-
call_combiner_.SetNotifyOnCancel(nullptr);
|
882
|
-
}
|
883
|
-
InternalUnref("destroy");
|
884
|
-
}
|
885
|
-
|
886
|
-
// start_batch_closure points to a caller-allocated closure to be used
|
887
|
-
// for entering the call combiner.
|
888
|
-
void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
889
|
-
grpc_closure* start_batch_closure) {
|
890
|
-
// This is called via the call combiner to start sending a batch down
|
891
|
-
// the filter stack.
|
892
|
-
auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
|
893
|
-
grpc_transport_stream_op_batch* batch =
|
894
|
-
static_cast<grpc_transport_stream_op_batch*>(arg);
|
895
|
-
auto* call =
|
896
|
-
static_cast<FilterStackCall*>(batch->handler_private.extra_arg);
|
897
|
-
grpc_call_element* elem = call->call_elem(0);
|
898
|
-
GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
|
899
|
-
elem->filter->start_transport_stream_op_batch(elem, batch);
|
900
|
-
};
|
901
|
-
batch->handler_private.extra_arg = this;
|
902
|
-
GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
|
903
|
-
grpc_schedule_on_exec_ctx);
|
904
|
-
GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
|
905
|
-
absl::OkStatus(), "executing batch");
|
906
|
-
}
|
907
|
-
|
908
|
-
namespace {
|
909
|
-
struct CancelState {
|
910
|
-
FilterStackCall* call;
|
911
|
-
grpc_closure start_batch;
|
912
|
-
grpc_closure finish_batch;
|
913
|
-
};
|
914
|
-
} // namespace
|
915
|
-
|
916
|
-
// The on_complete callback used when sending a cancel_stream batch down
|
917
|
-
// the filter stack. Yields the call combiner when the batch is done.
|
918
|
-
static void done_termination(void* arg, grpc_error_handle /*error*/) {
|
919
|
-
CancelState* state = static_cast<CancelState*>(arg);
|
920
|
-
GRPC_CALL_COMBINER_STOP(state->call->call_combiner(),
|
921
|
-
"on_complete for cancel_stream op");
|
922
|
-
state->call->InternalUnref("termination");
|
923
|
-
delete state;
|
924
|
-
}
|
925
|
-
|
926
|
-
void FilterStackCall::CancelWithError(grpc_error_handle error) {
|
927
|
-
if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
|
928
|
-
return;
|
929
|
-
}
|
930
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
931
|
-
gpr_log(GPR_INFO, "CancelWithError %s %s", is_client() ? "CLI" : "SVR",
|
932
|
-
StatusToString(error).c_str());
|
933
|
-
}
|
934
|
-
ClearPeerString();
|
935
|
-
InternalRef("termination");
|
936
|
-
ResetDeadline();
|
937
|
-
// Inform the call combiner of the cancellation, so that it can cancel
|
938
|
-
// any in-flight asynchronous actions that may be holding the call
|
939
|
-
// combiner. This ensures that the cancel_stream batch can be sent
|
940
|
-
// down the filter stack in a timely manner.
|
941
|
-
call_combiner_.Cancel(error);
|
942
|
-
CancelState* state = new CancelState;
|
943
|
-
state->call = this;
|
944
|
-
GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
|
945
|
-
grpc_schedule_on_exec_ctx);
|
946
|
-
grpc_transport_stream_op_batch* op =
|
947
|
-
grpc_make_transport_stream_op(&state->finish_batch);
|
948
|
-
op->cancel_stream = true;
|
949
|
-
op->payload->cancel_stream.cancel_error = error;
|
950
|
-
ExecuteBatch(op, &state->start_batch);
|
951
|
-
}
|
952
|
-
|
953
|
-
void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
954
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
955
|
-
gpr_log(GPR_INFO, "set_final_status %s %s", is_client() ? "CLI" : "SVR",
|
956
|
-
StatusToString(error).c_str());
|
957
|
-
}
|
958
|
-
ResetDeadline();
|
959
|
-
if (is_client()) {
|
960
|
-
std::string status_details;
|
961
|
-
grpc_error_get_status(error, send_deadline(), final_op_.client.status,
|
962
|
-
&status_details, nullptr,
|
963
|
-
final_op_.client.error_string);
|
964
|
-
*final_op_.client.status_details =
|
965
|
-
grpc_slice_from_cpp_string(std::move(status_details));
|
966
|
-
status_error_.set(error);
|
967
|
-
channelz::ChannelNode* channelz_channel = channel()->channelz_node();
|
968
|
-
if (channelz_channel != nullptr) {
|
969
|
-
if (*final_op_.client.status != GRPC_STATUS_OK) {
|
970
|
-
channelz_channel->RecordCallFailed();
|
971
|
-
} else {
|
972
|
-
channelz_channel->RecordCallSucceeded();
|
973
|
-
}
|
974
|
-
}
|
975
|
-
} else {
|
976
|
-
*final_op_.server.cancelled =
|
977
|
-
!error.ok() || !sent_server_trailing_metadata_;
|
978
|
-
channelz::ServerNode* channelz_node =
|
979
|
-
final_op_.server.core_server->channelz_node();
|
980
|
-
if (channelz_node != nullptr) {
|
981
|
-
if (*final_op_.server.cancelled || !status_error_.ok()) {
|
982
|
-
channelz_node->RecordCallFailed();
|
983
|
-
} else {
|
984
|
-
channelz_node->RecordCallSucceeded();
|
985
|
-
}
|
986
|
-
}
|
987
|
-
}
|
988
|
-
}
|
989
|
-
|
990
|
-
bool FilterStackCall::PrepareApplicationMetadata(size_t count,
|
991
|
-
grpc_metadata* metadata,
|
992
|
-
bool is_trailing) {
|
993
|
-
grpc_metadata_batch* batch =
|
994
|
-
is_trailing ? &send_trailing_metadata_ : &send_initial_metadata_;
|
995
|
-
for (size_t i = 0; i < count; i++) {
|
996
|
-
grpc_metadata* md = &metadata[i];
|
997
|
-
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
998
|
-
grpc_validate_header_key_is_legal(md->key))) {
|
999
|
-
return false;
|
1000
|
-
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1001
|
-
!GRPC_LOG_IF_ERROR(
|
1002
|
-
"validate_metadata",
|
1003
|
-
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1004
|
-
return false;
|
1005
|
-
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1006
|
-
// HTTP2 hpack encoding has a maximum limit.
|
1007
|
-
return false;
|
1008
|
-
} else if (grpc_slice_str_cmp(md->key, "content-length") == 0) {
|
1009
|
-
// Filter "content-length metadata"
|
1010
|
-
continue;
|
1011
|
-
}
|
1012
|
-
batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
|
1013
|
-
[md](absl::string_view error, const Slice& value) {
|
1014
|
-
gpr_log(GPR_DEBUG, "Append error: %s",
|
1015
|
-
absl::StrCat("key=", StringViewFromSlice(md->key),
|
1016
|
-
" error=", error,
|
1017
|
-
" value=", value.as_string_view())
|
1018
|
-
.c_str());
|
1019
|
-
});
|
1020
|
-
}
|
1021
|
-
|
1022
|
-
return true;
|
1023
|
-
}
|
1024
|
-
|
1025
|
-
namespace {
|
1026
|
-
class PublishToAppEncoder {
|
1027
|
-
public:
|
1028
|
-
explicit PublishToAppEncoder(grpc_metadata_array* dest,
|
1029
|
-
const grpc_metadata_batch* encoding,
|
1030
|
-
bool is_client)
|
1031
|
-
: dest_(dest), encoding_(encoding), is_client_(is_client) {}
|
1032
|
-
|
1033
|
-
void Encode(const Slice& key, const Slice& value) {
|
1034
|
-
Append(key.c_slice(), value.c_slice());
|
1035
|
-
}
|
1036
|
-
|
1037
|
-
// Catch anything that is not explicitly handled, and do not publish it to the
|
1038
|
-
// application. If new metadata is added to a batch that needs to be
|
1039
|
-
// published, it should be called out here.
|
1040
|
-
template <typename Which>
|
1041
|
-
void Encode(Which, const typename Which::ValueType&) {}
|
1042
|
-
|
1043
|
-
void Encode(UserAgentMetadata, const Slice& slice) {
|
1044
|
-
Append(UserAgentMetadata::key(), slice);
|
1045
|
-
}
|
1046
|
-
|
1047
|
-
void Encode(HostMetadata, const Slice& slice) {
|
1048
|
-
Append(HostMetadata::key(), slice);
|
1049
|
-
}
|
1050
|
-
|
1051
|
-
void Encode(GrpcPreviousRpcAttemptsMetadata, uint32_t count) {
|
1052
|
-
Append(GrpcPreviousRpcAttemptsMetadata::key(), count);
|
1053
|
-
}
|
1054
|
-
|
1055
|
-
void Encode(GrpcRetryPushbackMsMetadata, Duration count) {
|
1056
|
-
Append(GrpcRetryPushbackMsMetadata::key(), count.millis());
|
1057
|
-
}
|
1058
|
-
|
1059
|
-
void Encode(LbTokenMetadata, const Slice& slice) {
|
1060
|
-
Append(LbTokenMetadata::key(), slice);
|
1061
|
-
}
|
1062
|
-
|
1063
|
-
private:
|
1064
|
-
void Append(absl::string_view key, int64_t value) {
|
1065
|
-
Append(StaticSlice::FromStaticString(key).c_slice(),
|
1066
|
-
Slice::FromInt64(value).c_slice());
|
1067
|
-
}
|
1068
|
-
|
1069
|
-
void Append(absl::string_view key, const Slice& value) {
|
1070
|
-
Append(StaticSlice::FromStaticString(key).c_slice(), value.c_slice());
|
1071
|
-
}
|
1072
|
-
|
1073
|
-
void Append(grpc_slice key, grpc_slice value) {
|
1074
|
-
if (dest_->count == dest_->capacity) {
|
1075
|
-
Crash(absl::StrCat(
|
1076
|
-
"Too many metadata entries: capacity=", dest_->capacity, " on ",
|
1077
|
-
is_client_ ? "client" : "server", " encoding ", encoding_->count(),
|
1078
|
-
" elements: ", encoding_->DebugString().c_str()));
|
1079
|
-
}
|
1080
|
-
auto* mdusr = &dest_->metadata[dest_->count++];
|
1081
|
-
mdusr->key = key;
|
1082
|
-
mdusr->value = value;
|
1083
|
-
}
|
1084
|
-
|
1085
|
-
grpc_metadata_array* const dest_;
|
1086
|
-
const grpc_metadata_batch* const encoding_;
|
1087
|
-
const bool is_client_;
|
1088
|
-
};
|
1089
|
-
} // namespace
|
1090
|
-
|
1091
|
-
void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
|
1092
|
-
bool is_trailing) {
|
1093
|
-
if (b->count() == 0) return;
|
1094
|
-
if (!is_client() && is_trailing) return;
|
1095
|
-
if (is_trailing && buffered_metadata_[1] == nullptr) return;
|
1096
|
-
grpc_metadata_array* dest;
|
1097
|
-
dest = buffered_metadata_[is_trailing];
|
1098
|
-
if (dest->count + b->count() > dest->capacity) {
|
1099
|
-
dest->capacity =
|
1100
|
-
std::max(dest->capacity + b->count(), dest->capacity * 3 / 2);
|
1101
|
-
dest->metadata = static_cast<grpc_metadata*>(
|
1102
|
-
gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity));
|
1103
|
-
}
|
1104
|
-
PublishToAppEncoder encoder(dest, b, is_client());
|
1105
|
-
b->Encode(&encoder);
|
1106
|
-
}
|
1107
|
-
|
1108
|
-
void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
|
1109
|
-
ProcessIncomingInitialMetadata(*b);
|
1110
|
-
PublishAppMetadata(b, false);
|
1111
|
-
}
|
1112
|
-
|
1113
|
-
void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
1114
|
-
grpc_error_handle batch_error) {
|
1115
|
-
if (!batch_error.ok()) {
|
1116
|
-
SetFinalStatus(batch_error);
|
1117
|
-
} else {
|
1118
|
-
absl::optional<grpc_status_code> grpc_status =
|
1119
|
-
b->Take(GrpcStatusMetadata());
|
1120
|
-
if (grpc_status.has_value()) {
|
1121
|
-
grpc_status_code status_code = *grpc_status;
|
1122
|
-
grpc_error_handle error;
|
1123
|
-
if (status_code != GRPC_STATUS_OK) {
|
1124
|
-
Slice peer = GetPeerString();
|
1125
|
-
error = grpc_error_set_int(
|
1126
|
-
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
|
1127
|
-
peer.as_string_view())),
|
1128
|
-
StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
|
1129
|
-
}
|
1130
|
-
auto grpc_message = b->Take(GrpcMessageMetadata());
|
1131
|
-
if (grpc_message.has_value()) {
|
1132
|
-
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
|
1133
|
-
grpc_message->as_string_view());
|
1134
|
-
} else if (!error.ok()) {
|
1135
|
-
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
|
1136
|
-
}
|
1137
|
-
SetFinalStatus(error);
|
1138
|
-
} else if (!is_client()) {
|
1139
|
-
SetFinalStatus(absl::OkStatus());
|
1140
|
-
} else {
|
1141
|
-
gpr_log(GPR_DEBUG,
|
1142
|
-
"Received trailing metadata with no error and no status");
|
1143
|
-
SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
|
1144
|
-
StatusIntProperty::kRpcStatus,
|
1145
|
-
GRPC_STATUS_UNKNOWN));
|
1146
|
-
}
|
1147
|
-
}
|
1148
|
-
PublishAppMetadata(b, true);
|
1149
|
-
}
|
1150
|
-
|
1151
|
-
namespace {
|
1152
|
-
bool AreWriteFlagsValid(uint32_t flags) {
|
1153
|
-
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
1154
|
-
const uint32_t allowed_write_positions =
|
1155
|
-
(GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
|
1156
|
-
const uint32_t invalid_positions = ~allowed_write_positions;
|
1157
|
-
return !(flags & invalid_positions);
|
1158
|
-
}
|
1159
|
-
|
1160
|
-
bool AreInitialMetadataFlagsValid(uint32_t flags) {
|
1161
|
-
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
1162
|
-
uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
|
1163
|
-
return !(flags & invalid_positions);
|
1164
|
-
}
|
1165
|
-
|
1166
|
-
size_t BatchSlotForOp(grpc_op_type type) {
|
1167
|
-
switch (type) {
|
1168
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
1169
|
-
return 0;
|
1170
|
-
case GRPC_OP_SEND_MESSAGE:
|
1171
|
-
return 1;
|
1172
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
1173
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
1174
|
-
return 2;
|
1175
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
1176
|
-
return 3;
|
1177
|
-
case GRPC_OP_RECV_MESSAGE:
|
1178
|
-
return 4;
|
1179
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
1180
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
1181
|
-
return 5;
|
1182
|
-
}
|
1183
|
-
GPR_UNREACHABLE_CODE(return 123456789);
|
1184
|
-
}
|
1185
|
-
} // namespace
|
1186
|
-
|
1187
|
-
FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
|
1188
|
-
const grpc_op* ops) {
|
1189
|
-
size_t slot_idx = BatchSlotForOp(ops[0].op);
|
1190
|
-
BatchControl** pslot = &active_batches_[slot_idx];
|
1191
|
-
BatchControl* bctl;
|
1192
|
-
if (*pslot != nullptr) {
|
1193
|
-
bctl = *pslot;
|
1194
|
-
if (bctl->call_ != nullptr) {
|
1195
|
-
return nullptr;
|
1196
|
-
}
|
1197
|
-
bctl->~BatchControl();
|
1198
|
-
bctl->op_ = {};
|
1199
|
-
new (&bctl->batch_error_) AtomicError();
|
1200
|
-
} else {
|
1201
|
-
bctl = arena()->New<BatchControl>();
|
1202
|
-
*pslot = bctl;
|
1203
|
-
}
|
1204
|
-
bctl->call_ = this;
|
1205
|
-
bctl->call_tracer_ = static_cast<CallTracerAnnotationInterface*>(
|
1206
|
-
ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
|
1207
|
-
bctl->op_.payload = &stream_op_payload_;
|
1208
|
-
return bctl;
|
1209
|
-
}
|
1210
|
-
|
1211
|
-
void FilterStackCall::BatchControl::PostCompletion() {
|
1212
|
-
FilterStackCall* call = call_;
|
1213
|
-
grpc_error_handle error = batch_error_.get();
|
1214
|
-
|
1215
|
-
if (IsCallStatusOverrideOnCancellationEnabled()) {
|
1216
|
-
// On the client side, if final call status is already known (i.e if this op
|
1217
|
-
// includes recv_trailing_metadata) and if the call status is known to be
|
1218
|
-
// OK, then disregard the batch error to ensure call->receiving_buffer_ is
|
1219
|
-
// not cleared.
|
1220
|
-
if (op_.recv_trailing_metadata && call->is_client() &&
|
1221
|
-
call->status_error_.ok()) {
|
1222
|
-
error = absl::OkStatus();
|
1223
|
-
}
|
1224
|
-
}
|
1225
|
-
|
1226
|
-
if (grpc_call_trace.enabled()) {
|
1227
|
-
gpr_log(GPR_DEBUG, "tag:%p batch_error=%s op:%s",
|
1228
|
-
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1229
|
-
grpc_transport_stream_op_batch_string(&op_, false).c_str());
|
1230
|
-
}
|
1231
|
-
|
1232
|
-
if (op_.send_initial_metadata) {
|
1233
|
-
call->send_initial_metadata_.Clear();
|
1234
|
-
}
|
1235
|
-
if (op_.send_message) {
|
1236
|
-
if (op_.payload->send_message.stream_write_closed && error.ok()) {
|
1237
|
-
error = grpc_error_add_child(
|
1238
|
-
error, GRPC_ERROR_CREATE(
|
1239
|
-
"Attempt to send message after stream was closed."));
|
1240
|
-
}
|
1241
|
-
call->sending_message_ = false;
|
1242
|
-
call->send_slice_buffer_.Clear();
|
1243
|
-
}
|
1244
|
-
if (op_.send_trailing_metadata) {
|
1245
|
-
call->send_trailing_metadata_.Clear();
|
1246
|
-
}
|
1247
|
-
|
1248
|
-
if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
|
1249
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1250
|
-
*call->receiving_buffer_ = nullptr;
|
1251
|
-
}
|
1252
|
-
if (op_.recv_trailing_metadata) {
|
1253
|
-
// propagate cancellation to any interested children
|
1254
|
-
gpr_atm_rel_store(&call->received_final_op_atm_, 1);
|
1255
|
-
call->PropagateCancellationToChildren();
|
1256
|
-
error = absl::OkStatus();
|
1257
|
-
}
|
1258
|
-
batch_error_.set(absl::OkStatus());
|
1259
|
-
|
1260
|
-
if (completion_data_.notify_tag.is_closure) {
|
1261
|
-
call_ = nullptr;
|
1262
|
-
GrpcClosure::Run(
|
1263
|
-
DEBUG_LOCATION,
|
1264
|
-
static_cast<grpc_closure*>(completion_data_.notify_tag.tag), error);
|
1265
|
-
call->InternalUnref("completion");
|
1266
|
-
} else {
|
1267
|
-
grpc_cq_end_op(
|
1268
|
-
call->cq_, completion_data_.notify_tag.tag, error,
|
1269
|
-
[](void* user_data, grpc_cq_completion* /*storage*/) {
|
1270
|
-
BatchControl* bctl = static_cast<BatchControl*>(user_data);
|
1271
|
-
Call* call = bctl->call_;
|
1272
|
-
bctl->call_ = nullptr;
|
1273
|
-
call->InternalUnref("completion");
|
1274
|
-
},
|
1275
|
-
this, &completion_data_.cq_completion);
|
1276
|
-
}
|
1277
|
-
}
|
1278
|
-
|
1279
|
-
void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
|
1280
|
-
if (GPR_UNLIKELY(completed_batch_step(op))) {
|
1281
|
-
PostCompletion();
|
1282
|
-
}
|
1283
|
-
}
|
1284
|
-
|
1285
|
-
void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
|
1286
|
-
FilterStackCall* call = call_;
|
1287
|
-
if (!call->receiving_slice_buffer_.has_value()) {
|
1288
|
-
*call->receiving_buffer_ = nullptr;
|
1289
|
-
call->receiving_message_ = false;
|
1290
|
-
FinishStep(PendingOp::kRecvMessage);
|
1291
|
-
} else {
|
1292
|
-
call->NoteLastMessageFlags(call->receiving_stream_flags_);
|
1293
|
-
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
1294
|
-
(call->incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
1295
|
-
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
|
1296
|
-
nullptr, 0, call->incoming_compression_algorithm());
|
1297
|
-
} else {
|
1298
|
-
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
1299
|
-
}
|
1300
|
-
grpc_slice_buffer_move_into(
|
1301
|
-
call->receiving_slice_buffer_->c_slice_buffer(),
|
1302
|
-
&(*call->receiving_buffer_)->data.raw.slice_buffer);
|
1303
|
-
call->receiving_message_ = false;
|
1304
|
-
call->receiving_slice_buffer_.reset();
|
1305
|
-
FinishStep(PendingOp::kRecvMessage);
|
1306
|
-
}
|
1307
|
-
}
|
1308
|
-
|
1309
|
-
void FilterStackCall::BatchControl::ReceivingStreamReady(
|
1310
|
-
grpc_error_handle error) {
|
1311
|
-
if (grpc_call_trace.enabled()) {
|
1312
|
-
gpr_log(GPR_DEBUG,
|
1313
|
-
"tag:%p ReceivingStreamReady error=%s "
|
1314
|
-
"receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
|
1315
|
-
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1316
|
-
call_->receiving_slice_buffer_.has_value(),
|
1317
|
-
gpr_atm_no_barrier_load(&call_->recv_state_));
|
1318
|
-
}
|
1319
|
-
FilterStackCall* call = call_;
|
1320
|
-
if (!error.ok()) {
|
1321
|
-
call->receiving_slice_buffer_.reset();
|
1322
|
-
if (batch_error_.ok()) {
|
1323
|
-
batch_error_.set(error);
|
1324
|
-
}
|
1325
|
-
call->CancelWithError(error);
|
1326
|
-
}
|
1327
|
-
// If recv_state is kRecvNone, we will save the batch_control
|
1328
|
-
// object with rel_cas, and will not use it after the cas. Its corresponding
|
1329
|
-
// acq_load is in receiving_initial_metadata_ready()
|
1330
|
-
if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
|
1331
|
-
!gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
|
1332
|
-
reinterpret_cast<gpr_atm>(this))) {
|
1333
|
-
ProcessDataAfterMetadata();
|
1334
|
-
}
|
1335
|
-
}
|
1336
|
-
|
1337
|
-
void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
1338
|
-
grpc_error_handle error) {
|
1339
|
-
FilterStackCall* call = call_;
|
1340
|
-
|
1341
|
-
GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
|
1342
|
-
|
1343
|
-
if (error.ok()) {
|
1344
|
-
grpc_metadata_batch* md = &call->recv_initial_metadata_;
|
1345
|
-
call->RecvInitialFilter(md);
|
1346
|
-
|
1347
|
-
absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
|
1348
|
-
if (deadline.has_value() && !call->is_client()) {
|
1349
|
-
call_->set_send_deadline(*deadline);
|
1350
|
-
}
|
1351
|
-
} else {
|
1352
|
-
if (batch_error_.ok()) {
|
1353
|
-
batch_error_.set(error);
|
1354
|
-
}
|
1355
|
-
call->CancelWithError(error);
|
1356
|
-
}
|
384
|
+
} // namespace grpc_core
|
1357
385
|
|
1358
|
-
|
1359
|
-
|
1360
|
-
gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
|
1361
|
-
// Should only receive initial metadata once
|
1362
|
-
CHECK_NE(rsr_bctlp, 1);
|
1363
|
-
if (rsr_bctlp == 0) {
|
1364
|
-
// We haven't seen initial metadata and messages before, thus initial
|
1365
|
-
// metadata is received first.
|
1366
|
-
// no_barrier_cas is used, as this function won't access the batch_control
|
1367
|
-
// object saved by receiving_stream_ready() if the initial metadata is
|
1368
|
-
// received first.
|
1369
|
-
if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
|
1370
|
-
kRecvInitialMetadataFirst)) {
|
1371
|
-
break;
|
1372
|
-
}
|
1373
|
-
} else {
|
1374
|
-
// Already received messages
|
1375
|
-
saved_rsr_closure = GRPC_CLOSURE_CREATE(
|
1376
|
-
[](void* bctl, grpc_error_handle error) {
|
1377
|
-
static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
|
1378
|
-
},
|
1379
|
-
reinterpret_cast<BatchControl*>(rsr_bctlp),
|
1380
|
-
grpc_schedule_on_exec_ctx);
|
1381
|
-
// No need to modify recv_state
|
1382
|
-
break;
|
1383
|
-
}
|
1384
|
-
}
|
1385
|
-
if (saved_rsr_closure != nullptr) {
|
1386
|
-
GrpcClosure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
|
1387
|
-
}
|
386
|
+
///////////////////////////////////////////////////////////////////////////////
|
387
|
+
// C-based API
|
1388
388
|
|
1389
|
-
|
389
|
+
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
390
|
+
grpc_core::ExecCtx exec_ctx;
|
391
|
+
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
1390
392
|
}
|
1391
393
|
|
1392
|
-
void
|
1393
|
-
|
1394
|
-
|
1395
|
-
"recv_trailing_metadata_ready");
|
1396
|
-
grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
|
1397
|
-
call_->RecvTrailingFilter(md, error);
|
1398
|
-
FinishStep(PendingOp::kRecvTrailingMetadata);
|
394
|
+
void grpc_call_set_completion_queue(grpc_call* call,
|
395
|
+
grpc_completion_queue* cq) {
|
396
|
+
grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
|
1399
397
|
}
|
1400
398
|
|
1401
|
-
void
|
1402
|
-
GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "on_complete");
|
1403
|
-
if (batch_error_.ok()) {
|
1404
|
-
batch_error_.set(error);
|
1405
|
-
}
|
1406
|
-
if (!error.ok()) {
|
1407
|
-
call_->CancelWithError(error);
|
1408
|
-
}
|
1409
|
-
FinishStep(PendingOp::kSends);
|
1410
|
-
}
|
399
|
+
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
1411
400
|
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
if (!is_notify_tag_closure) {
|
1416
|
-
CHECK(grpc_cq_begin_op(cq, notify_tag));
|
1417
|
-
grpc_cq_end_op(
|
1418
|
-
cq, notify_tag, absl::OkStatus(),
|
1419
|
-
[](void*, grpc_cq_completion* completion) { gpr_free(completion); },
|
1420
|
-
nullptr,
|
1421
|
-
static_cast<grpc_cq_completion*>(
|
1422
|
-
gpr_malloc(sizeof(grpc_cq_completion))));
|
1423
|
-
} else {
|
1424
|
-
Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
|
1425
|
-
absl::OkStatus());
|
1426
|
-
}
|
1427
|
-
}
|
1428
|
-
} // namespace
|
1429
|
-
|
1430
|
-
grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
1431
|
-
void* notify_tag,
|
1432
|
-
bool is_notify_tag_closure) {
|
1433
|
-
size_t i;
|
1434
|
-
const grpc_op* op;
|
1435
|
-
BatchControl* bctl;
|
1436
|
-
grpc_call_error error = GRPC_CALL_OK;
|
1437
|
-
grpc_transport_stream_op_batch* stream_op;
|
1438
|
-
grpc_transport_stream_op_batch_payload* stream_op_payload;
|
1439
|
-
uint32_t seen_ops = 0;
|
1440
|
-
intptr_t pending_ops = 0;
|
1441
|
-
|
1442
|
-
for (i = 0; i < nops; i++) {
|
1443
|
-
if (seen_ops & (1u << ops[i].op)) {
|
1444
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1445
|
-
}
|
1446
|
-
seen_ops |= (1u << ops[i].op);
|
1447
|
-
}
|
1448
|
-
|
1449
|
-
if (!is_client() &&
|
1450
|
-
(seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
|
1451
|
-
(seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
|
1452
|
-
gpr_log(GPR_ERROR,
|
1453
|
-
"******************* SEND_STATUS WITH RECV_MESSAGE "
|
1454
|
-
"*******************");
|
1455
|
-
return GRPC_CALL_ERROR;
|
1456
|
-
}
|
1457
|
-
|
1458
|
-
GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
|
1459
|
-
|
1460
|
-
if (nops == 0) {
|
1461
|
-
EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
|
1462
|
-
error = GRPC_CALL_OK;
|
1463
|
-
goto done;
|
1464
|
-
}
|
1465
|
-
|
1466
|
-
bctl = ReuseOrAllocateBatchControl(ops);
|
1467
|
-
if (bctl == nullptr) {
|
1468
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1469
|
-
}
|
1470
|
-
bctl->completion_data_.notify_tag.tag = notify_tag;
|
1471
|
-
bctl->completion_data_.notify_tag.is_closure =
|
1472
|
-
static_cast<uint8_t>(is_notify_tag_closure != 0);
|
1473
|
-
|
1474
|
-
stream_op = &bctl->op_;
|
1475
|
-
stream_op_payload = &stream_op_payload_;
|
1476
|
-
|
1477
|
-
// rewrite batch ops into a transport op
|
1478
|
-
for (i = 0; i < nops; i++) {
|
1479
|
-
op = &ops[i];
|
1480
|
-
if (op->reserved != nullptr) {
|
1481
|
-
error = GRPC_CALL_ERROR;
|
1482
|
-
goto done_with_error;
|
1483
|
-
}
|
1484
|
-
switch (op->op) {
|
1485
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
1486
|
-
// Flag validation: currently allow no flags
|
1487
|
-
if (!AreInitialMetadataFlagsValid(op->flags)) {
|
1488
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1489
|
-
goto done_with_error;
|
1490
|
-
}
|
1491
|
-
if (sent_initial_metadata_) {
|
1492
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1493
|
-
goto done_with_error;
|
1494
|
-
}
|
1495
|
-
if (op->data.send_initial_metadata.count > INT_MAX) {
|
1496
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1497
|
-
goto done_with_error;
|
1498
|
-
}
|
1499
|
-
stream_op->send_initial_metadata = true;
|
1500
|
-
sent_initial_metadata_ = true;
|
1501
|
-
if (!PrepareApplicationMetadata(op->data.send_initial_metadata.count,
|
1502
|
-
op->data.send_initial_metadata.metadata,
|
1503
|
-
false)) {
|
1504
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1505
|
-
goto done_with_error;
|
1506
|
-
}
|
1507
|
-
PrepareOutgoingInitialMetadata(*op, send_initial_metadata_);
|
1508
|
-
// TODO(ctiller): just make these the same variable?
|
1509
|
-
if (is_client() && send_deadline() != Timestamp::InfFuture()) {
|
1510
|
-
send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
|
1511
|
-
}
|
1512
|
-
if (is_client()) {
|
1513
|
-
send_initial_metadata_.Set(
|
1514
|
-
WaitForReady(),
|
1515
|
-
WaitForReady::ValueType{
|
1516
|
-
(op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
1517
|
-
(op->flags &
|
1518
|
-
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
1519
|
-
}
|
1520
|
-
stream_op_payload->send_initial_metadata.send_initial_metadata =
|
1521
|
-
&send_initial_metadata_;
|
1522
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1523
|
-
break;
|
1524
|
-
}
|
1525
|
-
case GRPC_OP_SEND_MESSAGE: {
|
1526
|
-
if (!AreWriteFlagsValid(op->flags)) {
|
1527
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1528
|
-
goto done_with_error;
|
1529
|
-
}
|
1530
|
-
if (op->data.send_message.send_message == nullptr) {
|
1531
|
-
error = GRPC_CALL_ERROR_INVALID_MESSAGE;
|
1532
|
-
goto done_with_error;
|
1533
|
-
}
|
1534
|
-
if (sending_message_) {
|
1535
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1536
|
-
goto done_with_error;
|
1537
|
-
}
|
1538
|
-
uint32_t flags = op->flags;
|
1539
|
-
// If the outgoing buffer is already compressed, mark it as so in the
|
1540
|
-
// flags. These will be picked up by the compression filter and further
|
1541
|
-
// (wasteful) attempts at compression skipped.
|
1542
|
-
if (op->data.send_message.send_message->data.raw.compression >
|
1543
|
-
GRPC_COMPRESS_NONE) {
|
1544
|
-
flags |= GRPC_WRITE_INTERNAL_COMPRESS;
|
1545
|
-
}
|
1546
|
-
stream_op->send_message = true;
|
1547
|
-
sending_message_ = true;
|
1548
|
-
send_slice_buffer_.Clear();
|
1549
|
-
grpc_slice_buffer_move_into(
|
1550
|
-
&op->data.send_message.send_message->data.raw.slice_buffer,
|
1551
|
-
send_slice_buffer_.c_slice_buffer());
|
1552
|
-
stream_op_payload->send_message.flags = flags;
|
1553
|
-
stream_op_payload->send_message.send_message = &send_slice_buffer_;
|
1554
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1555
|
-
break;
|
1556
|
-
}
|
1557
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
|
1558
|
-
// Flag validation: currently allow no flags
|
1559
|
-
if (op->flags != 0) {
|
1560
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1561
|
-
goto done_with_error;
|
1562
|
-
}
|
1563
|
-
if (!is_client()) {
|
1564
|
-
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1565
|
-
goto done_with_error;
|
1566
|
-
}
|
1567
|
-
if (sent_final_op_) {
|
1568
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1569
|
-
goto done_with_error;
|
1570
|
-
}
|
1571
|
-
stream_op->send_trailing_metadata = true;
|
1572
|
-
sent_final_op_ = true;
|
1573
|
-
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1574
|
-
&send_trailing_metadata_;
|
1575
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1576
|
-
break;
|
1577
|
-
}
|
1578
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
1579
|
-
// Flag validation: currently allow no flags
|
1580
|
-
if (op->flags != 0) {
|
1581
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1582
|
-
goto done_with_error;
|
1583
|
-
}
|
1584
|
-
if (is_client()) {
|
1585
|
-
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1586
|
-
goto done_with_error;
|
1587
|
-
}
|
1588
|
-
if (sent_final_op_) {
|
1589
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1590
|
-
goto done_with_error;
|
1591
|
-
}
|
1592
|
-
if (op->data.send_status_from_server.trailing_metadata_count >
|
1593
|
-
INT_MAX) {
|
1594
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1595
|
-
goto done_with_error;
|
1596
|
-
}
|
1597
|
-
stream_op->send_trailing_metadata = true;
|
1598
|
-
sent_final_op_ = true;
|
1599
|
-
|
1600
|
-
if (!PrepareApplicationMetadata(
|
1601
|
-
op->data.send_status_from_server.trailing_metadata_count,
|
1602
|
-
op->data.send_status_from_server.trailing_metadata, true)) {
|
1603
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1604
|
-
goto done_with_error;
|
1605
|
-
}
|
1606
|
-
|
1607
|
-
grpc_error_handle status_error =
|
1608
|
-
op->data.send_status_from_server.status == GRPC_STATUS_OK
|
1609
|
-
? absl::OkStatus()
|
1610
|
-
: grpc_error_set_int(
|
1611
|
-
GRPC_ERROR_CREATE("Server returned error"),
|
1612
|
-
StatusIntProperty::kRpcStatus,
|
1613
|
-
static_cast<intptr_t>(
|
1614
|
-
op->data.send_status_from_server.status));
|
1615
|
-
if (op->data.send_status_from_server.status_details != nullptr) {
|
1616
|
-
send_trailing_metadata_.Set(
|
1617
|
-
GrpcMessageMetadata(),
|
1618
|
-
Slice(grpc_slice_copy(
|
1619
|
-
*op->data.send_status_from_server.status_details)));
|
1620
|
-
if (!status_error.ok()) {
|
1621
|
-
status_error = grpc_error_set_str(
|
1622
|
-
status_error, StatusStrProperty::kGrpcMessage,
|
1623
|
-
StringViewFromSlice(
|
1624
|
-
*op->data.send_status_from_server.status_details));
|
1625
|
-
}
|
1626
|
-
}
|
1627
|
-
|
1628
|
-
status_error_.set(status_error);
|
1629
|
-
|
1630
|
-
send_trailing_metadata_.Set(GrpcStatusMetadata(),
|
1631
|
-
op->data.send_status_from_server.status);
|
1632
|
-
|
1633
|
-
// Ignore any te metadata key value pairs specified.
|
1634
|
-
send_trailing_metadata_.Remove(TeMetadata());
|
1635
|
-
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1636
|
-
&send_trailing_metadata_;
|
1637
|
-
stream_op_payload->send_trailing_metadata.sent =
|
1638
|
-
&sent_server_trailing_metadata_;
|
1639
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1640
|
-
break;
|
1641
|
-
}
|
1642
|
-
case GRPC_OP_RECV_INITIAL_METADATA: {
|
1643
|
-
// Flag validation: currently allow no flags
|
1644
|
-
if (op->flags != 0) {
|
1645
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1646
|
-
goto done_with_error;
|
1647
|
-
}
|
1648
|
-
if (received_initial_metadata_) {
|
1649
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1650
|
-
goto done_with_error;
|
1651
|
-
}
|
1652
|
-
received_initial_metadata_ = true;
|
1653
|
-
buffered_metadata_[0] =
|
1654
|
-
op->data.recv_initial_metadata.recv_initial_metadata;
|
1655
|
-
GRPC_CLOSURE_INIT(
|
1656
|
-
&receiving_initial_metadata_ready_,
|
1657
|
-
[](void* bctl, grpc_error_handle error) {
|
1658
|
-
static_cast<BatchControl*>(bctl)->ReceivingInitialMetadataReady(
|
1659
|
-
error);
|
1660
|
-
},
|
1661
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1662
|
-
stream_op->recv_initial_metadata = true;
|
1663
|
-
stream_op_payload->recv_initial_metadata.recv_initial_metadata =
|
1664
|
-
&recv_initial_metadata_;
|
1665
|
-
stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
|
1666
|
-
&receiving_initial_metadata_ready_;
|
1667
|
-
if (is_client()) {
|
1668
|
-
stream_op_payload->recv_initial_metadata.trailing_metadata_available =
|
1669
|
-
&is_trailers_only_;
|
1670
|
-
}
|
1671
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
|
1672
|
-
break;
|
1673
|
-
}
|
1674
|
-
case GRPC_OP_RECV_MESSAGE: {
|
1675
|
-
// Flag validation: currently allow no flags
|
1676
|
-
if (op->flags != 0) {
|
1677
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1678
|
-
goto done_with_error;
|
1679
|
-
}
|
1680
|
-
if (receiving_message_) {
|
1681
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1682
|
-
goto done_with_error;
|
1683
|
-
}
|
1684
|
-
receiving_message_ = true;
|
1685
|
-
stream_op->recv_message = true;
|
1686
|
-
receiving_slice_buffer_.reset();
|
1687
|
-
receiving_buffer_ = op->data.recv_message.recv_message;
|
1688
|
-
stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
|
1689
|
-
receiving_stream_flags_ = 0;
|
1690
|
-
stream_op_payload->recv_message.flags = &receiving_stream_flags_;
|
1691
|
-
stream_op_payload->recv_message.call_failed_before_recv_message =
|
1692
|
-
&call_failed_before_recv_message_;
|
1693
|
-
GRPC_CLOSURE_INIT(
|
1694
|
-
&receiving_stream_ready_,
|
1695
|
-
[](void* bctlp, grpc_error_handle error) {
|
1696
|
-
auto* bctl = static_cast<BatchControl*>(bctlp);
|
1697
|
-
auto* call = bctl->call_;
|
1698
|
-
// Yields the call combiner before processing the received
|
1699
|
-
// message.
|
1700
|
-
GRPC_CALL_COMBINER_STOP(call->call_combiner(),
|
1701
|
-
"recv_message_ready");
|
1702
|
-
bctl->ReceivingStreamReady(error);
|
1703
|
-
},
|
1704
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1705
|
-
stream_op_payload->recv_message.recv_message_ready =
|
1706
|
-
&receiving_stream_ready_;
|
1707
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
|
1708
|
-
break;
|
1709
|
-
}
|
1710
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
1711
|
-
// Flag validation: currently allow no flags
|
1712
|
-
if (op->flags != 0) {
|
1713
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1714
|
-
goto done_with_error;
|
1715
|
-
}
|
1716
|
-
if (!is_client()) {
|
1717
|
-
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1718
|
-
goto done_with_error;
|
1719
|
-
}
|
1720
|
-
if (requested_final_op_) {
|
1721
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1722
|
-
goto done_with_error;
|
1723
|
-
}
|
1724
|
-
requested_final_op_ = true;
|
1725
|
-
buffered_metadata_[1] =
|
1726
|
-
op->data.recv_status_on_client.trailing_metadata;
|
1727
|
-
final_op_.client.status = op->data.recv_status_on_client.status;
|
1728
|
-
final_op_.client.status_details =
|
1729
|
-
op->data.recv_status_on_client.status_details;
|
1730
|
-
final_op_.client.error_string =
|
1731
|
-
op->data.recv_status_on_client.error_string;
|
1732
|
-
stream_op->recv_trailing_metadata = true;
|
1733
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1734
|
-
&recv_trailing_metadata_;
|
1735
|
-
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1736
|
-
&final_info_.stats.transport_stream_stats;
|
1737
|
-
GRPC_CLOSURE_INIT(
|
1738
|
-
&receiving_trailing_metadata_ready_,
|
1739
|
-
[](void* bctl, grpc_error_handle error) {
|
1740
|
-
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1741
|
-
error);
|
1742
|
-
},
|
1743
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1744
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1745
|
-
&receiving_trailing_metadata_ready_;
|
1746
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1747
|
-
break;
|
1748
|
-
}
|
1749
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER: {
|
1750
|
-
// Flag validation: currently allow no flags
|
1751
|
-
if (op->flags != 0) {
|
1752
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1753
|
-
goto done_with_error;
|
1754
|
-
}
|
1755
|
-
if (is_client()) {
|
1756
|
-
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1757
|
-
goto done_with_error;
|
1758
|
-
}
|
1759
|
-
if (requested_final_op_) {
|
1760
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1761
|
-
goto done_with_error;
|
1762
|
-
}
|
1763
|
-
requested_final_op_ = true;
|
1764
|
-
final_op_.server.cancelled = op->data.recv_close_on_server.cancelled;
|
1765
|
-
stream_op->recv_trailing_metadata = true;
|
1766
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1767
|
-
&recv_trailing_metadata_;
|
1768
|
-
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1769
|
-
&final_info_.stats.transport_stream_stats;
|
1770
|
-
GRPC_CLOSURE_INIT(
|
1771
|
-
&receiving_trailing_metadata_ready_,
|
1772
|
-
[](void* bctl, grpc_error_handle error) {
|
1773
|
-
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1774
|
-
error);
|
1775
|
-
},
|
1776
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1777
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1778
|
-
&receiving_trailing_metadata_ready_;
|
1779
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1780
|
-
break;
|
1781
|
-
}
|
1782
|
-
}
|
1783
|
-
}
|
1784
|
-
|
1785
|
-
InternalRef("completion");
|
1786
|
-
if (!is_notify_tag_closure) {
|
1787
|
-
CHECK(grpc_cq_begin_op(cq_, notify_tag));
|
1788
|
-
}
|
1789
|
-
bctl->set_pending_ops(pending_ops);
|
1790
|
-
|
1791
|
-
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
1792
|
-
GRPC_CLOSURE_INIT(
|
1793
|
-
&bctl->finish_batch_,
|
1794
|
-
[](void* bctl, grpc_error_handle error) {
|
1795
|
-
static_cast<BatchControl*>(bctl)->FinishBatch(error);
|
1796
|
-
},
|
1797
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1798
|
-
stream_op->on_complete = &bctl->finish_batch_;
|
1799
|
-
}
|
1800
|
-
|
1801
|
-
if (grpc_call_trace.enabled()) {
|
1802
|
-
gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1803
|
-
PendingOpString(pending_ops).c_str(),
|
1804
|
-
grpc_transport_stream_op_batch_string(stream_op, false).c_str(),
|
1805
|
-
bctl->completion_data_.notify_tag.tag);
|
1806
|
-
}
|
1807
|
-
ExecuteBatch(stream_op, &bctl->start_batch_);
|
1808
|
-
|
1809
|
-
done:
|
1810
|
-
return error;
|
1811
|
-
|
1812
|
-
done_with_error:
|
1813
|
-
// reverse any mutations that occurred
|
1814
|
-
if (stream_op->send_initial_metadata) {
|
1815
|
-
sent_initial_metadata_ = false;
|
1816
|
-
send_initial_metadata_.Clear();
|
1817
|
-
}
|
1818
|
-
if (stream_op->send_message) {
|
1819
|
-
sending_message_ = false;
|
1820
|
-
}
|
1821
|
-
if (stream_op->send_trailing_metadata) {
|
1822
|
-
sent_final_op_ = false;
|
1823
|
-
send_trailing_metadata_.Clear();
|
1824
|
-
}
|
1825
|
-
if (stream_op->recv_initial_metadata) {
|
1826
|
-
received_initial_metadata_ = false;
|
1827
|
-
}
|
1828
|
-
if (stream_op->recv_message) {
|
1829
|
-
receiving_message_ = false;
|
1830
|
-
}
|
1831
|
-
if (stream_op->recv_trailing_metadata) {
|
1832
|
-
requested_final_op_ = false;
|
1833
|
-
}
|
1834
|
-
goto done;
|
1835
|
-
}
|
1836
|
-
|
1837
|
-
void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
|
1838
|
-
void (*destroy)(void*)) {
|
1839
|
-
if (context_[elem].destroy) {
|
1840
|
-
context_[elem].destroy(context_[elem].value);
|
1841
|
-
}
|
1842
|
-
context_[elem].value = value;
|
1843
|
-
context_[elem].destroy = destroy;
|
1844
|
-
}
|
1845
|
-
|
1846
|
-
///////////////////////////////////////////////////////////////////////////////
|
1847
|
-
// Metadata validation helpers
|
1848
|
-
|
1849
|
-
namespace {
|
1850
|
-
bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
|
1851
|
-
if (count > INT_MAX) {
|
1852
|
-
return false;
|
1853
|
-
}
|
1854
|
-
for (size_t i = 0; i < count; i++) {
|
1855
|
-
grpc_metadata* md = &metadata[i];
|
1856
|
-
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
1857
|
-
grpc_validate_header_key_is_legal(md->key))) {
|
1858
|
-
return false;
|
1859
|
-
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1860
|
-
!GRPC_LOG_IF_ERROR(
|
1861
|
-
"validate_metadata",
|
1862
|
-
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1863
|
-
return false;
|
1864
|
-
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1865
|
-
// HTTP2 hpack encoding has a maximum limit.
|
1866
|
-
return false;
|
1867
|
-
}
|
1868
|
-
}
|
1869
|
-
return true;
|
1870
|
-
}
|
1871
|
-
} // namespace
|
1872
|
-
|
1873
|
-
///////////////////////////////////////////////////////////////////////////////
|
1874
|
-
// PromiseBasedCall
|
1875
|
-
// Will be folded into Call once the promise conversion is done
|
1876
|
-
|
1877
|
-
class BasicPromiseBasedCall : public Call, public Party {
|
1878
|
-
public:
|
1879
|
-
using Call::arena;
|
1880
|
-
|
1881
|
-
BasicPromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
1882
|
-
uint32_t initial_internal_refs,
|
1883
|
-
const grpc_call_create_args& args)
|
1884
|
-
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
1885
|
-
args.channel->Ref()),
|
1886
|
-
Party(initial_internal_refs),
|
1887
|
-
external_refs_(initial_external_refs),
|
1888
|
-
cq_(args.cq) {
|
1889
|
-
if (args.cq != nullptr) {
|
1890
|
-
GRPC_CQ_INTERNAL_REF(args.cq, "bind");
|
1891
|
-
}
|
1892
|
-
context_[GRPC_CONTEXT_CALL].value = this;
|
1893
|
-
}
|
1894
|
-
|
1895
|
-
~BasicPromiseBasedCall() override {
|
1896
|
-
if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
|
1897
|
-
for (int i = 0; i < GRPC_CONTEXT_COUNT; i++) {
|
1898
|
-
if (context_[i].destroy) {
|
1899
|
-
context_[i].destroy(context_[i].value);
|
1900
|
-
}
|
1901
|
-
}
|
1902
|
-
}
|
1903
|
-
|
1904
|
-
virtual void OrphanCall() = 0;
|
1905
|
-
|
1906
|
-
virtual ServerCallContext* server_call_context() { return nullptr; }
|
1907
|
-
void SetCompletionQueue(grpc_completion_queue* cq) final {
|
1908
|
-
cq_ = cq;
|
1909
|
-
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
1910
|
-
}
|
1911
|
-
|
1912
|
-
// Implementation of call refcounting: move this to DualRefCounted once we
|
1913
|
-
// don't need to maintain FilterStackCall compatibility
|
1914
|
-
void ExternalRef() final {
|
1915
|
-
if (external_refs_.fetch_add(1, std::memory_order_relaxed) == 0) {
|
1916
|
-
InternalRef("external");
|
1917
|
-
}
|
1918
|
-
}
|
1919
|
-
void ExternalUnref() final {
|
1920
|
-
if (external_refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
|
1921
|
-
OrphanCall();
|
1922
|
-
InternalUnref("external");
|
1923
|
-
}
|
1924
|
-
}
|
1925
|
-
void InternalRef(const char* reason) final {
|
1926
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1927
|
-
gpr_log(GPR_DEBUG, "INTERNAL_REF:%p:%s", this, reason);
|
1928
|
-
}
|
1929
|
-
Party::IncrementRefCount();
|
1930
|
-
}
|
1931
|
-
void InternalUnref(const char* reason) final {
|
1932
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1933
|
-
gpr_log(GPR_DEBUG, "INTERNAL_UNREF:%p:%s", this, reason);
|
1934
|
-
}
|
1935
|
-
Party::Unref();
|
1936
|
-
}
|
1937
|
-
|
1938
|
-
void RunInContext(absl::AnyInvocable<void()> fn) {
|
1939
|
-
Spawn(
|
1940
|
-
"run_in_context",
|
1941
|
-
[fn = std::move(fn)]() mutable {
|
1942
|
-
fn();
|
1943
|
-
return Empty{};
|
1944
|
-
},
|
1945
|
-
[](Empty) {});
|
1946
|
-
}
|
1947
|
-
|
1948
|
-
void ContextSet(grpc_context_index elem, void* value,
|
1949
|
-
void (*destroy)(void*)) final {
|
1950
|
-
if (context_[elem].destroy != nullptr) {
|
1951
|
-
context_[elem].destroy(context_[elem].value);
|
1952
|
-
}
|
1953
|
-
context_[elem].value = value;
|
1954
|
-
context_[elem].destroy = destroy;
|
1955
|
-
}
|
1956
|
-
|
1957
|
-
void* ContextGet(grpc_context_index elem) const final {
|
1958
|
-
return context_[elem].value;
|
1959
|
-
}
|
1960
|
-
|
1961
|
-
// Accept the stats from the context (call once we have proof the transport is
|
1962
|
-
// done with them).
|
1963
|
-
void AcceptTransportStatsFromContext() {
|
1964
|
-
final_stats_ = *call_context_.call_stats();
|
1965
|
-
}
|
1966
|
-
|
1967
|
-
// This should return nullptr for the promise stack (and alternative means
|
1968
|
-
// for that functionality be invented)
|
1969
|
-
grpc_call_stack* call_stack() final { return nullptr; }
|
1970
|
-
|
1971
|
-
virtual RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs) {
|
1972
|
-
Crash("Not implemented");
|
1973
|
-
}
|
1974
|
-
|
1975
|
-
protected:
|
1976
|
-
class ScopedContext
|
1977
|
-
: public ScopedActivity,
|
1978
|
-
public promise_detail::Context<Arena>,
|
1979
|
-
public promise_detail::Context<grpc_call_context_element>,
|
1980
|
-
public promise_detail::Context<CallContext>,
|
1981
|
-
public promise_detail::Context<CallFinalization> {
|
1982
|
-
public:
|
1983
|
-
explicit ScopedContext(BasicPromiseBasedCall* call)
|
1984
|
-
: ScopedActivity(call),
|
1985
|
-
promise_detail::Context<Arena>(call->arena()),
|
1986
|
-
promise_detail::Context<grpc_call_context_element>(call->context_),
|
1987
|
-
promise_detail::Context<CallContext>(&call->call_context_),
|
1988
|
-
promise_detail::Context<CallFinalization>(&call->finalization_) {}
|
1989
|
-
};
|
1990
|
-
|
1991
|
-
grpc_call_context_element* context() { return context_; }
|
1992
|
-
|
1993
|
-
grpc_completion_queue* cq() { return cq_; }
|
1994
|
-
|
1995
|
-
// At the end of the call run any finalization actions.
|
1996
|
-
void SetFinalizationStatus(grpc_status_code status, Slice status_details) {
|
1997
|
-
final_message_ = std::move(status_details);
|
1998
|
-
final_status_ = status;
|
1999
|
-
}
|
2000
|
-
|
2001
|
-
grpc_event_engine::experimental::EventEngine* event_engine() const override {
|
2002
|
-
return channel()->event_engine();
|
2003
|
-
}
|
2004
|
-
|
2005
|
-
private:
|
2006
|
-
void PartyOver() final {
|
2007
|
-
{
|
2008
|
-
ScopedContext ctx(this);
|
2009
|
-
std::string message;
|
2010
|
-
grpc_call_final_info final_info;
|
2011
|
-
final_info.stats = final_stats_;
|
2012
|
-
final_info.final_status = final_status_;
|
2013
|
-
// TODO(ctiller): change type here so we don't need to copy this string.
|
2014
|
-
final_info.error_string = nullptr;
|
2015
|
-
if (!final_message_.empty()) {
|
2016
|
-
message = std::string(final_message_.begin(), final_message_.end());
|
2017
|
-
final_info.error_string = message.c_str();
|
2018
|
-
}
|
2019
|
-
final_info.stats.latency =
|
2020
|
-
gpr_cycle_counter_sub(gpr_get_cycle_counter(), start_time());
|
2021
|
-
finalization_.Run(&final_info);
|
2022
|
-
CancelRemainingParticipants();
|
2023
|
-
arena()->DestroyManagedNewObjects();
|
2024
|
-
}
|
2025
|
-
DeleteThis();
|
2026
|
-
}
|
2027
|
-
|
2028
|
-
// Double refcounted for now: party owns the internal refcount, we track the
|
2029
|
-
// external refcount. Figure out a better scheme post-promise conversion.
|
2030
|
-
std::atomic<size_t> external_refs_;
|
2031
|
-
CallFinalization finalization_;
|
2032
|
-
CallContext call_context_{this};
|
2033
|
-
// Contexts for various subsystems (security, tracing, ...).
|
2034
|
-
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
2035
|
-
grpc_call_stats final_stats_{};
|
2036
|
-
Slice final_message_;
|
2037
|
-
grpc_status_code final_status_ = GRPC_STATUS_UNKNOWN;
|
2038
|
-
grpc_completion_queue* cq_;
|
2039
|
-
};
|
2040
|
-
|
2041
|
-
class PromiseBasedCall : public BasicPromiseBasedCall {
|
2042
|
-
public:
|
2043
|
-
PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2044
|
-
const grpc_call_create_args& args);
|
2045
|
-
|
2046
|
-
bool Completed() final { return finished_.IsSet(); }
|
2047
|
-
|
2048
|
-
bool failed_before_recv_message() const final {
|
2049
|
-
return failed_before_recv_message_.load(std::memory_order_relaxed);
|
2050
|
-
}
|
2051
|
-
|
2052
|
-
using Call::arena;
|
2053
|
-
|
2054
|
-
protected:
|
2055
|
-
class ScopedContext : public BasicPromiseBasedCall::ScopedContext,
|
2056
|
-
public BatchBuilder,
|
2057
|
-
public promise_detail::Context<BatchBuilder> {
|
2058
|
-
public:
|
2059
|
-
explicit ScopedContext(PromiseBasedCall* call)
|
2060
|
-
: BasicPromiseBasedCall::ScopedContext(call),
|
2061
|
-
BatchBuilder(&call->batch_payload_),
|
2062
|
-
promise_detail::Context<BatchBuilder>(this) {}
|
2063
|
-
};
|
2064
|
-
|
2065
|
-
class Completion {
|
2066
|
-
public:
|
2067
|
-
Completion() : index_(kNullIndex) {}
|
2068
|
-
~Completion() { CHECK(index_ == kNullIndex); }
|
2069
|
-
explicit Completion(uint8_t index) : index_(index) {}
|
2070
|
-
Completion(const Completion& other) = delete;
|
2071
|
-
Completion& operator=(const Completion& other) = delete;
|
2072
|
-
Completion(Completion&& other) noexcept : index_(other.index_) {
|
2073
|
-
other.index_ = kNullIndex;
|
2074
|
-
}
|
2075
|
-
Completion& operator=(Completion&& other) noexcept {
|
2076
|
-
CHECK(index_ == kNullIndex);
|
2077
|
-
index_ = other.index_;
|
2078
|
-
other.index_ = kNullIndex;
|
2079
|
-
return *this;
|
2080
|
-
}
|
2081
|
-
|
2082
|
-
uint8_t index() const { return index_; }
|
2083
|
-
uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
|
2084
|
-
bool has_value() const { return index_ != kNullIndex; }
|
2085
|
-
|
2086
|
-
private:
|
2087
|
-
enum : uint8_t { kNullIndex = 0xff };
|
2088
|
-
uint8_t index_;
|
2089
|
-
};
|
2090
|
-
|
2091
|
-
// Enumerates why a Completion is still pending
|
2092
|
-
enum class PendingOp {
|
2093
|
-
// We're in the midst of starting a batch of operations
|
2094
|
-
kStartingBatch = 0,
|
2095
|
-
// The following correspond with the batch operations from above
|
2096
|
-
kSendInitialMetadata,
|
2097
|
-
kReceiveInitialMetadata,
|
2098
|
-
kReceiveStatusOnClient,
|
2099
|
-
kReceiveCloseOnServer = kReceiveStatusOnClient,
|
2100
|
-
kSendMessage,
|
2101
|
-
kReceiveMessage,
|
2102
|
-
kSendStatusFromServer,
|
2103
|
-
kSendCloseFromClient = kSendStatusFromServer,
|
2104
|
-
};
|
2105
|
-
|
2106
|
-
bool RunParty() override {
|
2107
|
-
ScopedContext ctx(this);
|
2108
|
-
return Party::RunParty();
|
2109
|
-
}
|
2110
|
-
|
2111
|
-
const char* PendingOpString(PendingOp reason) const {
|
2112
|
-
switch (reason) {
|
2113
|
-
case PendingOp::kStartingBatch:
|
2114
|
-
return "StartingBatch";
|
2115
|
-
case PendingOp::kSendInitialMetadata:
|
2116
|
-
return "SendInitialMetadata";
|
2117
|
-
case PendingOp::kReceiveInitialMetadata:
|
2118
|
-
return "ReceiveInitialMetadata";
|
2119
|
-
case PendingOp::kReceiveStatusOnClient:
|
2120
|
-
return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
|
2121
|
-
case PendingOp::kSendMessage:
|
2122
|
-
return "SendMessage";
|
2123
|
-
case PendingOp::kReceiveMessage:
|
2124
|
-
return "ReceiveMessage";
|
2125
|
-
case PendingOp::kSendStatusFromServer:
|
2126
|
-
return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
|
2127
|
-
}
|
2128
|
-
return "Unknown";
|
2129
|
-
}
|
2130
|
-
|
2131
|
-
static constexpr uint32_t PendingOpBit(PendingOp reason) {
|
2132
|
-
return 1 << static_cast<int>(reason);
|
2133
|
-
}
|
2134
|
-
|
2135
|
-
// Begin work on a completion, recording the tag/closure to notify.
|
2136
|
-
// Use the op selected in \a ops to determine the index to allocate into.
|
2137
|
-
// Starts the "StartingBatch" PendingOp immediately.
|
2138
|
-
// Assumes at least one operation in \a ops.
|
2139
|
-
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops);
|
2140
|
-
// Add one pending op to the completion, and return it.
|
2141
|
-
Completion AddOpToCompletion(const Completion& completion, PendingOp reason);
|
2142
|
-
// Stringify a completion
|
2143
|
-
std::string CompletionString(const Completion& completion) const {
|
2144
|
-
return completion.has_value()
|
2145
|
-
? completion_info_[completion.index()].pending.ToString(this)
|
2146
|
-
: "no-completion";
|
2147
|
-
}
|
2148
|
-
// Finish one op on the completion. Must have been previously been added.
|
2149
|
-
// The completion as a whole finishes when all pending ops finish.
|
2150
|
-
void FinishOpOnCompletion(Completion* completion, PendingOp reason);
|
2151
|
-
// Mark the completion as failed. Does not finish it.
|
2152
|
-
void FailCompletion(const Completion& completion,
|
2153
|
-
SourceLocation source_location = {});
|
2154
|
-
// Mark the completion as infallible. Overrides FailCompletion to report
|
2155
|
-
// success always.
|
2156
|
-
void ForceCompletionSuccess(const Completion& completion);
|
2157
|
-
|
2158
|
-
std::string PresentAndCompletionText(const char* caption, bool has,
|
2159
|
-
const Completion& completion) const {
|
2160
|
-
if (has) {
|
2161
|
-
if (completion.has_value()) {
|
2162
|
-
return absl::StrCat(caption, ":", CompletionString(completion), " ");
|
2163
|
-
} else {
|
2164
|
-
return absl::StrCat(caption,
|
2165
|
-
":!!BUG:operation is present, no completion!! ");
|
2166
|
-
}
|
2167
|
-
} else {
|
2168
|
-
if (!completion.has_value()) {
|
2169
|
-
return "";
|
2170
|
-
} else {
|
2171
|
-
return absl::StrCat(caption, ":no-op:", CompletionString(completion),
|
2172
|
-
" ");
|
2173
|
-
}
|
2174
|
-
}
|
2175
|
-
}
|
2176
|
-
|
2177
|
-
// Spawn a job that will first do FirstPromise then receive a message
|
2178
|
-
template <typename FirstPromise>
|
2179
|
-
void StartRecvMessage(const grpc_op& op, const Completion& completion,
|
2180
|
-
FirstPromise first,
|
2181
|
-
PipeReceiver<MessageHandle>* receiver,
|
2182
|
-
bool cancel_on_error, Party::BulkSpawner& spawner);
|
2183
|
-
void StartSendMessage(const grpc_op& op, const Completion& completion,
|
2184
|
-
PipeSender<MessageHandle>* sender,
|
2185
|
-
Party::BulkSpawner& spawner);
|
2186
|
-
|
2187
|
-
void set_completed() { finished_.Set(); }
|
2188
|
-
|
2189
|
-
// Returns a promise that resolves to Empty whenever the call is completed.
|
2190
|
-
auto finished() { return finished_.Wait(); }
|
2191
|
-
|
2192
|
-
// Returns a promise that resolves to Empty whenever there is no outstanding
|
2193
|
-
// send operation
|
2194
|
-
auto WaitForSendingStarted() {
|
2195
|
-
return [this]() -> Poll<Empty> {
|
2196
|
-
int n = sends_queued_.load(std::memory_order_relaxed);
|
2197
|
-
if (grpc_call_trace.enabled()) {
|
2198
|
-
gpr_log(GPR_DEBUG, "%s[call] WaitForSendingStarted n=%d",
|
2199
|
-
DebugTag().c_str(), n);
|
2200
|
-
}
|
2201
|
-
if (n != 0) return waiting_for_queued_sends_.pending();
|
2202
|
-
return Empty{};
|
2203
|
-
};
|
2204
|
-
}
|
2205
|
-
|
2206
|
-
// Mark that a send has been queued - blocks sending trailing metadata.
|
2207
|
-
void QueueSend() {
|
2208
|
-
if (grpc_call_trace.enabled()) {
|
2209
|
-
gpr_log(GPR_DEBUG, "%s[call] QueueSend", DebugTag().c_str());
|
2210
|
-
}
|
2211
|
-
sends_queued_.fetch_add(1, std::memory_order_relaxed);
|
2212
|
-
}
|
2213
|
-
// Mark that a send has been dequeued - allows sending trailing metadata once
|
2214
|
-
// zero sends are queued.
|
2215
|
-
void EnactSend() {
|
2216
|
-
if (grpc_call_trace.enabled()) {
|
2217
|
-
gpr_log(GPR_DEBUG, "%s[call] EnactSend", DebugTag().c_str());
|
2218
|
-
}
|
2219
|
-
if (1 == sends_queued_.fetch_sub(1, std::memory_order_relaxed)) {
|
2220
|
-
waiting_for_queued_sends_.Wake();
|
2221
|
-
}
|
2222
|
-
}
|
2223
|
-
|
2224
|
-
void set_failed_before_recv_message() {
|
2225
|
-
failed_before_recv_message_.store(true, std::memory_order_relaxed);
|
2226
|
-
}
|
2227
|
-
|
2228
|
-
private:
|
2229
|
-
union CompletionInfo {
|
2230
|
-
static constexpr uint32_t kOpFailed = 0x8000'0000u;
|
2231
|
-
static constexpr uint32_t kOpForceSuccess = 0x4000'0000u;
|
2232
|
-
CompletionInfo() {}
|
2233
|
-
enum CompletionState {
|
2234
|
-
kPending,
|
2235
|
-
kSuccess,
|
2236
|
-
kFailure,
|
2237
|
-
};
|
2238
|
-
struct Pending {
|
2239
|
-
// Bitmask of PendingOps at the bottom, and kOpFailed, kOpForceSuccess at
|
2240
|
-
// the top.
|
2241
|
-
std::atomic<uint32_t> state;
|
2242
|
-
bool is_closure;
|
2243
|
-
// True if this completion was for a recv_message op.
|
2244
|
-
// In that case if the completion as a whole fails we need to cleanup the
|
2245
|
-
// returned message.
|
2246
|
-
bool is_recv_message;
|
2247
|
-
void* tag;
|
2248
|
-
|
2249
|
-
void Start(bool is_closure, void* tag) {
|
2250
|
-
this->is_closure = is_closure;
|
2251
|
-
this->is_recv_message = false;
|
2252
|
-
this->tag = tag;
|
2253
|
-
state.store(PendingOpBit(PendingOp::kStartingBatch),
|
2254
|
-
std::memory_order_release);
|
2255
|
-
}
|
2256
|
-
|
2257
|
-
void AddPendingBit(PendingOp reason) {
|
2258
|
-
if (reason == PendingOp::kReceiveMessage) is_recv_message = true;
|
2259
|
-
auto prev =
|
2260
|
-
state.fetch_or(PendingOpBit(reason), std::memory_order_relaxed);
|
2261
|
-
CHECK_EQ((prev & PendingOpBit(reason)), 0u);
|
2262
|
-
}
|
2263
|
-
|
2264
|
-
CompletionState RemovePendingBit(PendingOp reason) {
|
2265
|
-
const uint32_t mask = ~PendingOpBit(reason);
|
2266
|
-
auto prev = state.fetch_and(mask, std::memory_order_acq_rel);
|
2267
|
-
CHECK_NE((prev & PendingOpBit(reason)), 0u);
|
2268
|
-
switch (prev & mask) {
|
2269
|
-
case kOpFailed:
|
2270
|
-
return kFailure;
|
2271
|
-
case kOpFailed | kOpForceSuccess:
|
2272
|
-
case kOpForceSuccess:
|
2273
|
-
case 0:
|
2274
|
-
return kSuccess;
|
2275
|
-
default:
|
2276
|
-
return kPending;
|
2277
|
-
}
|
2278
|
-
}
|
2279
|
-
|
2280
|
-
void MarkFailed() {
|
2281
|
-
state.fetch_or(kOpFailed, std::memory_order_relaxed);
|
2282
|
-
}
|
2283
|
-
|
2284
|
-
void MarkForceSuccess() {
|
2285
|
-
state.fetch_or(kOpForceSuccess, std::memory_order_relaxed);
|
2286
|
-
}
|
2287
|
-
|
2288
|
-
std::string ToString(const PromiseBasedCall* call) const {
|
2289
|
-
auto state = this->state.load(std::memory_order_relaxed);
|
2290
|
-
std::vector<absl::string_view> pending_ops;
|
2291
|
-
for (size_t i = 0; i < 24; i++) {
|
2292
|
-
if (state & (1u << i)) {
|
2293
|
-
pending_ops.push_back(
|
2294
|
-
call->PendingOpString(static_cast<PendingOp>(i)));
|
2295
|
-
}
|
2296
|
-
}
|
2297
|
-
return absl::StrFormat("{%s}%s:tag=%p", absl::StrJoin(pending_ops, ","),
|
2298
|
-
(state & kOpForceSuccess) ? ":force-success"
|
2299
|
-
: (state & kOpFailed) ? ":failed"
|
2300
|
-
: ":success",
|
2301
|
-
tag);
|
2302
|
-
}
|
2303
|
-
} pending;
|
2304
|
-
grpc_cq_completion completion;
|
2305
|
-
};
|
2306
|
-
|
2307
|
-
CompletionInfo completion_info_[6];
|
2308
|
-
ExternallyObservableLatch<void> finished_;
|
2309
|
-
// Non-zero with an outstanding GRPC_OP_SEND_INITIAL_METADATA or
|
2310
|
-
// GRPC_OP_SEND_MESSAGE (one count each), and 0 once those payloads have been
|
2311
|
-
// pushed onto the outgoing pipe.
|
2312
|
-
std::atomic<uint8_t> sends_queued_{0};
|
2313
|
-
std::atomic<bool> failed_before_recv_message_{false};
|
2314
|
-
// Waiter for when sends_queued_ becomes 0.
|
2315
|
-
IntraActivityWaiter waiting_for_queued_sends_;
|
2316
|
-
grpc_byte_buffer** recv_message_ = nullptr;
|
2317
|
-
grpc_transport_stream_op_batch_payload batch_payload_{context()};
|
2318
|
-
};
|
2319
|
-
|
2320
|
-
template <typename T>
|
2321
|
-
grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
|
2322
|
-
grpc_call** out_call) {
|
2323
|
-
Channel* channel = args->channel.get();
|
2324
|
-
|
2325
|
-
auto* arena = channel->CreateArena();
|
2326
|
-
PromiseBasedCall* call = arena->New<T>(arena, args);
|
2327
|
-
*out_call = call->c_ptr();
|
2328
|
-
DCHECK(Call::FromC(*out_call) == call);
|
2329
|
-
return absl::OkStatus();
|
2330
|
-
}
|
2331
|
-
|
2332
|
-
PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2333
|
-
const grpc_call_create_args& args)
|
2334
|
-
: BasicPromiseBasedCall(arena, initial_external_refs,
|
2335
|
-
initial_external_refs != 0 ? 1 : 0, args) {}
|
2336
|
-
|
2337
|
-
static void CToMetadata(grpc_metadata* metadata, size_t count,
|
2338
|
-
grpc_metadata_batch* b) {
|
2339
|
-
for (size_t i = 0; i < count; i++) {
|
2340
|
-
grpc_metadata* md = &metadata[i];
|
2341
|
-
auto key = StringViewFromSlice(md->key);
|
2342
|
-
// Filter "content-length metadata"
|
2343
|
-
if (key == "content-length") continue;
|
2344
|
-
b->Append(key, Slice(CSliceRef(md->value)),
|
2345
|
-
[md](absl::string_view error, const Slice& value) {
|
2346
|
-
gpr_log(GPR_DEBUG, "Append error: %s",
|
2347
|
-
absl::StrCat("key=", StringViewFromSlice(md->key),
|
2348
|
-
" error=", error,
|
2349
|
-
" value=", value.as_string_view())
|
2350
|
-
.c_str());
|
2351
|
-
});
|
2352
|
-
}
|
2353
|
-
}
|
2354
|
-
|
2355
|
-
PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
|
2356
|
-
void* tag, bool is_closure, const grpc_op* ops) {
|
2357
|
-
Completion c(BatchSlotForOp(ops[0].op));
|
2358
|
-
if (!is_closure) {
|
2359
|
-
grpc_cq_begin_op(cq(), tag);
|
2360
|
-
}
|
2361
|
-
completion_info_[c.index()].pending.Start(is_closure, tag);
|
2362
|
-
if (grpc_call_trace.enabled()) {
|
2363
|
-
gpr_log(GPR_INFO, "%s[call] StartCompletion %s", DebugTag().c_str(),
|
2364
|
-
CompletionString(c).c_str());
|
2365
|
-
}
|
2366
|
-
return c;
|
2367
|
-
}
|
2368
|
-
|
2369
|
-
PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
|
2370
|
-
const Completion& completion, PendingOp reason) {
|
2371
|
-
if (grpc_call_trace.enabled()) {
|
2372
|
-
gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
|
2373
|
-
CompletionString(completion).c_str(), PendingOpString(reason));
|
2374
|
-
}
|
2375
|
-
CHECK(completion.has_value());
|
2376
|
-
completion_info_[completion.index()].pending.AddPendingBit(reason);
|
2377
|
-
return Completion(completion.index());
|
2378
|
-
}
|
2379
|
-
|
2380
|
-
void PromiseBasedCall::FailCompletion(const Completion& completion,
|
2381
|
-
SourceLocation location) {
|
2382
|
-
if (grpc_call_trace.enabled()) {
|
2383
|
-
gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
|
2384
|
-
"%s[call] FailCompletion %s", DebugTag().c_str(),
|
2385
|
-
CompletionString(completion).c_str());
|
2386
|
-
}
|
2387
|
-
completion_info_[completion.index()].pending.MarkFailed();
|
2388
|
-
}
|
2389
|
-
|
2390
|
-
void PromiseBasedCall::ForceCompletionSuccess(const Completion& completion) {
|
2391
|
-
completion_info_[completion.index()].pending.MarkForceSuccess();
|
2392
|
-
}
|
2393
|
-
|
2394
|
-
void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
|
2395
|
-
PendingOp reason) {
|
2396
|
-
if (grpc_call_trace.enabled()) {
|
2397
|
-
gpr_log(GPR_INFO, "%s[call] FinishOpOnCompletion completion:%s finish:%s",
|
2398
|
-
DebugTag().c_str(), CompletionString(*completion).c_str(),
|
2399
|
-
PendingOpString(reason));
|
2400
|
-
}
|
2401
|
-
const uint8_t i = completion->TakeIndex();
|
2402
|
-
CHECK(i < GPR_ARRAY_SIZE(completion_info_));
|
2403
|
-
CompletionInfo::Pending& pending = completion_info_[i].pending;
|
2404
|
-
bool success;
|
2405
|
-
switch (pending.RemovePendingBit(reason)) {
|
2406
|
-
case CompletionInfo::kPending:
|
2407
|
-
return; // Early out
|
2408
|
-
case CompletionInfo::kSuccess:
|
2409
|
-
success = true;
|
2410
|
-
break;
|
2411
|
-
case CompletionInfo::kFailure:
|
2412
|
-
success = false;
|
2413
|
-
break;
|
2414
|
-
}
|
2415
|
-
if (pending.is_recv_message && !success && *recv_message_ != nullptr) {
|
2416
|
-
grpc_byte_buffer_destroy(*recv_message_);
|
2417
|
-
*recv_message_ = nullptr;
|
2418
|
-
}
|
2419
|
-
auto error = success ? absl::OkStatus() : absl::CancelledError();
|
2420
|
-
if (pending.is_closure) {
|
2421
|
-
ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
|
2422
|
-
error);
|
2423
|
-
} else {
|
2424
|
-
InternalRef("cq_end_op");
|
2425
|
-
grpc_cq_end_op(
|
2426
|
-
cq(), pending.tag, error,
|
2427
|
-
[](void* p, grpc_cq_completion*) {
|
2428
|
-
static_cast<PromiseBasedCall*>(p)->InternalUnref("cq_end_op");
|
2429
|
-
},
|
2430
|
-
this, &completion_info_[i].completion);
|
2431
|
-
}
|
2432
|
-
}
|
2433
|
-
|
2434
|
-
void PromiseBasedCall::StartSendMessage(const grpc_op& op,
|
2435
|
-
const Completion& completion,
|
2436
|
-
PipeSender<MessageHandle>* sender,
|
2437
|
-
Party::BulkSpawner& spawner) {
|
2438
|
-
QueueSend();
|
2439
|
-
SliceBuffer send;
|
2440
|
-
grpc_slice_buffer_swap(
|
2441
|
-
&op.data.send_message.send_message->data.raw.slice_buffer,
|
2442
|
-
send.c_slice_buffer());
|
2443
|
-
auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
|
2444
|
-
spawner.Spawn(
|
2445
|
-
"call_send_message",
|
2446
|
-
[this, sender, msg = std::move(msg)]() mutable {
|
2447
|
-
EnactSend();
|
2448
|
-
return sender->Push(std::move(msg));
|
2449
|
-
},
|
2450
|
-
[this, completion = AddOpToCompletion(
|
2451
|
-
completion, PendingOp::kSendMessage)](bool result) mutable {
|
2452
|
-
if (grpc_call_trace.enabled()) {
|
2453
|
-
gpr_log(GPR_DEBUG, "%sSendMessage completes %s", DebugTag().c_str(),
|
2454
|
-
result ? "successfully" : "with failure");
|
2455
|
-
}
|
2456
|
-
if (!result) FailCompletion(completion);
|
2457
|
-
FinishOpOnCompletion(&completion, PendingOp::kSendMessage);
|
2458
|
-
});
|
2459
|
-
}
|
2460
|
-
|
2461
|
-
template <typename FirstPromiseFactory>
|
2462
|
-
void PromiseBasedCall::StartRecvMessage(
|
2463
|
-
const grpc_op& op, const Completion& completion,
|
2464
|
-
FirstPromiseFactory first_promise_factory,
|
2465
|
-
PipeReceiver<MessageHandle>* receiver, bool cancel_on_error,
|
2466
|
-
Party::BulkSpawner& spawner) {
|
2467
|
-
if (grpc_call_trace.enabled()) {
|
2468
|
-
gpr_log(GPR_INFO, "%s[call] Start RecvMessage: %s", DebugTag().c_str(),
|
2469
|
-
CompletionString(completion).c_str());
|
2470
|
-
}
|
2471
|
-
recv_message_ = op.data.recv_message.recv_message;
|
2472
|
-
spawner.Spawn(
|
2473
|
-
"call_recv_message",
|
2474
|
-
[first_promise_factory = std::move(first_promise_factory), receiver]() {
|
2475
|
-
return Seq(first_promise_factory(), receiver->Next());
|
2476
|
-
},
|
2477
|
-
[this, cancel_on_error,
|
2478
|
-
completion = AddOpToCompletion(completion, PendingOp::kReceiveMessage)](
|
2479
|
-
NextResult<MessageHandle> result) mutable {
|
2480
|
-
if (result.has_value()) {
|
2481
|
-
MessageHandle& message = *result;
|
2482
|
-
NoteLastMessageFlags(message->flags());
|
2483
|
-
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
2484
|
-
(incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
2485
|
-
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
2486
|
-
nullptr, 0, incoming_compression_algorithm());
|
2487
|
-
} else {
|
2488
|
-
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
2489
|
-
}
|
2490
|
-
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
2491
|
-
&(*recv_message_)->data.raw.slice_buffer);
|
2492
|
-
if (grpc_call_trace.enabled()) {
|
2493
|
-
gpr_log(GPR_INFO,
|
2494
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2495
|
-
"finishes: received %" PRIdPTR " byte message",
|
2496
|
-
DebugTag().c_str(),
|
2497
|
-
(*recv_message_)->data.raw.slice_buffer.length);
|
2498
|
-
}
|
2499
|
-
} else if (result.cancelled()) {
|
2500
|
-
if (grpc_call_trace.enabled()) {
|
2501
|
-
gpr_log(GPR_INFO,
|
2502
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2503
|
-
"finishes: received end-of-stream with error",
|
2504
|
-
DebugTag().c_str());
|
2505
|
-
}
|
2506
|
-
set_failed_before_recv_message();
|
2507
|
-
FailCompletion(completion);
|
2508
|
-
if (cancel_on_error) CancelWithError(absl::CancelledError());
|
2509
|
-
*recv_message_ = nullptr;
|
2510
|
-
} else {
|
2511
|
-
if (grpc_call_trace.enabled()) {
|
2512
|
-
gpr_log(GPR_INFO,
|
2513
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2514
|
-
"finishes: received end-of-stream",
|
2515
|
-
DebugTag().c_str());
|
2516
|
-
}
|
2517
|
-
*recv_message_ = nullptr;
|
2518
|
-
}
|
2519
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveMessage);
|
2520
|
-
});
|
2521
|
-
}
|
2522
|
-
|
2523
|
-
///////////////////////////////////////////////////////////////////////////////
|
2524
|
-
// CallContext
|
2525
|
-
|
2526
|
-
void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
|
2527
|
-
call_->RunInContext(std::move(fn));
|
2528
|
-
}
|
2529
|
-
|
2530
|
-
void CallContext::IncrementRefCount(const char* reason) {
|
2531
|
-
call_->InternalRef(reason);
|
2532
|
-
}
|
2533
|
-
|
2534
|
-
void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
|
2535
|
-
|
2536
|
-
ServerCallContext* CallContext::server_call_context() {
|
2537
|
-
return call_->server_call_context();
|
2538
|
-
}
|
2539
|
-
|
2540
|
-
RefCountedPtr<CallSpineInterface> CallContext::MakeCallSpine(
|
2541
|
-
CallArgs call_args) {
|
2542
|
-
return call_->MakeCallSpine(std::move(call_args));
|
2543
|
-
}
|
2544
|
-
|
2545
|
-
grpc_call* CallContext::c_call() { return call_->c_ptr(); }
|
2546
|
-
|
2547
|
-
///////////////////////////////////////////////////////////////////////////////
|
2548
|
-
// PublishMetadataArray
|
2549
|
-
|
2550
|
-
namespace {
|
2551
|
-
void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array,
|
2552
|
-
bool is_client) {
|
2553
|
-
const auto md_count = md->count();
|
2554
|
-
if (md_count > array->capacity) {
|
2555
|
-
array->capacity =
|
2556
|
-
std::max(array->capacity + md->count(), array->capacity * 3 / 2);
|
2557
|
-
array->metadata = static_cast<grpc_metadata*>(
|
2558
|
-
gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
|
2559
|
-
}
|
2560
|
-
PublishToAppEncoder encoder(array, md, is_client);
|
2561
|
-
md->Encode(&encoder);
|
2562
|
-
}
|
2563
|
-
} // namespace
|
2564
|
-
|
2565
|
-
///////////////////////////////////////////////////////////////////////////////
|
2566
|
-
// ClientPromiseBasedCall
|
2567
|
-
|
2568
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
2569
|
-
class ClientPromiseBasedCall final : public PromiseBasedCall {
|
2570
|
-
public:
|
2571
|
-
ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
|
2572
|
-
: PromiseBasedCall(arena, 1, *args),
|
2573
|
-
polling_entity_(
|
2574
|
-
args->cq != nullptr
|
2575
|
-
? grpc_polling_entity_create_from_pollset(
|
2576
|
-
grpc_cq_pollset(args->cq))
|
2577
|
-
: (args->pollset_set_alternative != nullptr
|
2578
|
-
? grpc_polling_entity_create_from_pollset_set(
|
2579
|
-
args->pollset_set_alternative)
|
2580
|
-
: grpc_polling_entity{})) {
|
2581
|
-
global_stats().IncrementClientCallsCreated();
|
2582
|
-
if (args->cq != nullptr) {
|
2583
|
-
CHECK(args->pollset_set_alternative == nullptr)
|
2584
|
-
<< "Only one of 'cq' and 'pollset_set_alternative' should be "
|
2585
|
-
"non-nullptr.";
|
2586
|
-
}
|
2587
|
-
ScopedContext context(this);
|
2588
|
-
args->channel->channel_stack()->stats_plugin_group->AddClientCallTracers(
|
2589
|
-
*args->path, args->registered_method, this->context());
|
2590
|
-
send_initial_metadata_ = Arena::MakePooled<ClientMetadata>();
|
2591
|
-
send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
|
2592
|
-
if (args->authority.has_value()) {
|
2593
|
-
send_initial_metadata_->Set(HttpAuthorityMetadata(),
|
2594
|
-
std::move(*args->authority));
|
2595
|
-
}
|
2596
|
-
send_initial_metadata_->Set(GrpcRegisteredMethod(),
|
2597
|
-
reinterpret_cast<void*>(static_cast<uintptr_t>(
|
2598
|
-
args->registered_method)));
|
2599
|
-
if (auto* channelz_channel = channel()->channelz_node()) {
|
2600
|
-
channelz_channel->RecordCallStarted();
|
2601
|
-
}
|
2602
|
-
if (args->send_deadline != Timestamp::InfFuture()) {
|
2603
|
-
UpdateDeadline(args->send_deadline);
|
2604
|
-
}
|
2605
|
-
Call* parent = Call::FromC(args->parent);
|
2606
|
-
if (parent != nullptr) {
|
2607
|
-
auto parent_status = InitParent(parent, args->propagation_mask);
|
2608
|
-
if (!parent_status.ok()) {
|
2609
|
-
CancelWithError(std::move(parent_status));
|
2610
|
-
}
|
2611
|
-
PublishToParent(parent);
|
2612
|
-
}
|
2613
|
-
}
|
2614
|
-
|
2615
|
-
void OrphanCall() override { MaybeUnpublishFromParent(); }
|
2616
|
-
|
2617
|
-
~ClientPromiseBasedCall() override {
|
2618
|
-
ScopedContext context(this);
|
2619
|
-
send_initial_metadata_.reset();
|
2620
|
-
// Need to destroy the pipes under the ScopedContext above, so we
|
2621
|
-
// move them out here and then allow the destructors to run at
|
2622
|
-
// end of scope, but before context.
|
2623
|
-
auto c2s = std::move(client_to_server_messages_);
|
2624
|
-
auto s2c = std::move(server_to_client_messages_);
|
2625
|
-
auto sim = std::move(server_initial_metadata_);
|
2626
|
-
}
|
2627
|
-
|
2628
|
-
void CancelWithError(absl::Status error) override {
|
2629
|
-
if (cancel_with_error_called_.exchange(true, std::memory_order_relaxed)) {
|
2630
|
-
return;
|
2631
|
-
}
|
2632
|
-
if (!started_.exchange(true, std::memory_order_relaxed)) {
|
2633
|
-
// Initial metadata not sent yet, so we can just fail the call.
|
2634
|
-
Spawn(
|
2635
|
-
"cancel_before_initial_metadata",
|
2636
|
-
[error = std::move(error), this]() {
|
2637
|
-
server_to_client_messages_.sender.Close();
|
2638
|
-
auto md = ServerMetadataFromStatus(error);
|
2639
|
-
md->Set(GrpcCallWasCancelled(), true);
|
2640
|
-
Finish(std::move(md));
|
2641
|
-
return Empty{};
|
2642
|
-
},
|
2643
|
-
[](Empty) {});
|
2644
|
-
} else {
|
2645
|
-
Spawn(
|
2646
|
-
"cancel_with_error",
|
2647
|
-
[error = std::move(error), this]() {
|
2648
|
-
if (!cancel_error_.is_set()) {
|
2649
|
-
auto md = ServerMetadataFromStatus(error);
|
2650
|
-
md->Set(GrpcCallWasCancelled(), true);
|
2651
|
-
cancel_error_.Set(std::move(md));
|
2652
|
-
}
|
2653
|
-
return Empty{};
|
2654
|
-
},
|
2655
|
-
[](Empty) {});
|
2656
|
-
}
|
2657
|
-
}
|
2658
|
-
absl::string_view GetServerAuthority() const override { abort(); }
|
2659
|
-
bool is_trailers_only() const override { return is_trailers_only_; }
|
2660
|
-
|
2661
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
2662
|
-
bool is_notify_tag_closure) override;
|
2663
|
-
|
2664
|
-
std::string DebugTag() const override {
|
2665
|
-
return absl::StrFormat("CLIENT_CALL[%p]: ", this);
|
2666
|
-
}
|
2667
|
-
|
2668
|
-
RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs call_args) final {
|
2669
|
-
class WrappingCallSpine final : public PipeBasedCallSpine {
|
2670
|
-
public:
|
2671
|
-
WrappingCallSpine(ClientPromiseBasedCall* call,
|
2672
|
-
ClientMetadataHandle metadata)
|
2673
|
-
: call_(call) {
|
2674
|
-
call_->InternalRef("call-spine");
|
2675
|
-
SpawnInfallible(
|
2676
|
-
"send_client_initial_metadata",
|
2677
|
-
[self = Ref(), metadata = std::move(metadata)]() mutable {
|
2678
|
-
return Map(self->client_initial_metadata_.sender.Push(
|
2679
|
-
std::move(metadata)),
|
2680
|
-
[self](bool) { return Empty{}; });
|
2681
|
-
});
|
2682
|
-
}
|
2683
|
-
|
2684
|
-
~WrappingCallSpine() override {
|
2685
|
-
{
|
2686
|
-
ScopedContext context(call_);
|
2687
|
-
// Move these out and destroy before the internal unref.
|
2688
|
-
auto client_initial_metadata = std::move(client_initial_metadata_);
|
2689
|
-
auto server_trailing_metadata = std::move(server_trailing_metadata_);
|
2690
|
-
}
|
2691
|
-
call_->InternalUnref("call-spine");
|
2692
|
-
}
|
2693
|
-
|
2694
|
-
Pipe<ClientMetadataHandle>& client_initial_metadata() override {
|
2695
|
-
return client_initial_metadata_;
|
2696
|
-
}
|
2697
|
-
|
2698
|
-
Pipe<MessageHandle>& client_to_server_messages() override {
|
2699
|
-
return call_->client_to_server_messages_;
|
2700
|
-
}
|
2701
|
-
|
2702
|
-
Pipe<ServerMetadataHandle>& server_initial_metadata() override {
|
2703
|
-
return call_->server_initial_metadata_;
|
2704
|
-
}
|
2705
|
-
|
2706
|
-
Pipe<MessageHandle>& server_to_client_messages() override {
|
2707
|
-
return call_->server_to_client_messages_;
|
2708
|
-
}
|
2709
|
-
|
2710
|
-
Latch<ServerMetadataHandle>& cancel_latch() override {
|
2711
|
-
return cancel_error_;
|
2712
|
-
}
|
2713
|
-
|
2714
|
-
Latch<bool>& was_cancelled_latch() override {
|
2715
|
-
return was_cancelled_latch_;
|
2716
|
-
}
|
2717
|
-
|
2718
|
-
Party& party() override { return *call_; }
|
2719
|
-
Arena* arena() override { return call_->arena(); }
|
2720
|
-
|
2721
|
-
void IncrementRefCount() override { refs_.Ref(); }
|
2722
|
-
void Unref() override {
|
2723
|
-
if (refs_.Unref()) delete this;
|
2724
|
-
}
|
2725
|
-
RefCountedPtr<WrappingCallSpine> Ref() {
|
2726
|
-
IncrementRefCount();
|
2727
|
-
return RefCountedPtr<WrappingCallSpine>(this);
|
2728
|
-
}
|
2729
|
-
|
2730
|
-
ClientMetadata& UnprocessedClientInitialMetadata() override {
|
2731
|
-
Crash("not for v2");
|
2732
|
-
}
|
2733
|
-
|
2734
|
-
void V2HackToStartCallWithoutACallFilterStack() override {}
|
2735
|
-
|
2736
|
-
private:
|
2737
|
-
RefCount refs_;
|
2738
|
-
ClientPromiseBasedCall* const call_;
|
2739
|
-
std::atomic<bool> sent_trailing_metadata_{false};
|
2740
|
-
Pipe<ClientMetadataHandle> client_initial_metadata_{call_->arena()};
|
2741
|
-
Pipe<ServerMetadataHandle> server_trailing_metadata_{call_->arena()};
|
2742
|
-
Latch<ServerMetadataHandle> cancel_error_;
|
2743
|
-
Latch<bool> was_cancelled_latch_;
|
2744
|
-
};
|
2745
|
-
CHECK(call_args.server_initial_metadata ==
|
2746
|
-
&server_initial_metadata_.sender);
|
2747
|
-
CHECK(call_args.client_to_server_messages ==
|
2748
|
-
&client_to_server_messages_.receiver);
|
2749
|
-
CHECK(call_args.server_to_client_messages ==
|
2750
|
-
&server_to_client_messages_.sender);
|
2751
|
-
call_args.client_initial_metadata_outstanding.Complete(true);
|
2752
|
-
return MakeRefCounted<WrappingCallSpine>(
|
2753
|
-
this, std::move(call_args.client_initial_metadata));
|
2754
|
-
}
|
2755
|
-
|
2756
|
-
private:
|
2757
|
-
// Finish the call with the given status/trailing metadata.
|
2758
|
-
void Finish(ServerMetadataHandle trailing_metadata);
|
2759
|
-
// Validate that a set of ops is valid for a client call.
|
2760
|
-
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
2761
|
-
// Commit a valid batch of operations to be executed.
|
2762
|
-
void CommitBatch(const grpc_op* ops, size_t nops,
|
2763
|
-
const Completion& completion);
|
2764
|
-
// Start the underlying promise.
|
2765
|
-
void StartPromise(ClientMetadataHandle client_initial_metadata,
|
2766
|
-
const Completion& completion, Party::BulkSpawner& spawner);
|
2767
|
-
// Start receiving initial metadata
|
2768
|
-
void StartRecvInitialMetadata(grpc_metadata_array* array,
|
2769
|
-
const Completion& completion,
|
2770
|
-
Party::BulkSpawner& spawner);
|
2771
|
-
void StartRecvStatusOnClient(
|
2772
|
-
const Completion& completion,
|
2773
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2774
|
-
Party::BulkSpawner& spawner);
|
2775
|
-
// Publish status out to the application.
|
2776
|
-
void PublishStatus(
|
2777
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2778
|
-
ServerMetadataHandle trailing_metadata);
|
2779
|
-
// Publish server initial metadata out to the application.
|
2780
|
-
void PublishInitialMetadata(ServerMetadata* metadata);
|
2781
|
-
|
2782
|
-
ClientMetadataHandle send_initial_metadata_;
|
2783
|
-
Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
|
2784
|
-
Latch<ServerMetadataHandle> server_trailing_metadata_;
|
2785
|
-
Latch<ServerMetadataHandle> cancel_error_;
|
2786
|
-
Latch<grpc_polling_entity> polling_entity_;
|
2787
|
-
Pipe<MessageHandle> client_to_server_messages_{arena()};
|
2788
|
-
Pipe<MessageHandle> server_to_client_messages_{arena()};
|
2789
|
-
bool is_trailers_only_ = false;
|
2790
|
-
bool scheduled_receive_status_ = false;
|
2791
|
-
bool scheduled_send_close_ = false;
|
2792
|
-
// True once the promise for the call is started.
|
2793
|
-
// This corresponds to sending initial metadata, or cancelling before doing
|
2794
|
-
// so.
|
2795
|
-
// In the latter case real world code sometimes does not sent the initial
|
2796
|
-
// metadata, and so gating based upon that does not work out.
|
2797
|
-
std::atomic<bool> started_{false};
|
2798
|
-
// True after the first CancelWithError call - prevents spamming cancels from
|
2799
|
-
// overflowing the party.
|
2800
|
-
std::atomic<bool> cancel_with_error_called_{false};
|
2801
|
-
// TODO(ctiller): delete when we remove the filter based API (may require some
|
2802
|
-
// cleanup in wrapped languages: they depend on this to hold slice refs)
|
2803
|
-
ServerMetadataHandle recv_initial_metadata_;
|
2804
|
-
ServerMetadataHandle recv_trailing_metadata_;
|
2805
|
-
};
|
2806
|
-
|
2807
|
-
void ClientPromiseBasedCall::StartPromise(
|
2808
|
-
ClientMetadataHandle client_initial_metadata, const Completion& completion,
|
2809
|
-
Party::BulkSpawner& spawner) {
|
2810
|
-
auto token = ClientInitialMetadataOutstandingToken::New(arena());
|
2811
|
-
spawner.Spawn(
|
2812
|
-
"call_send_initial_metadata", token.Wait(),
|
2813
|
-
[this,
|
2814
|
-
completion = AddOpToCompletion(
|
2815
|
-
completion, PendingOp::kSendInitialMetadata)](bool result) mutable {
|
2816
|
-
if (!result) FailCompletion(completion);
|
2817
|
-
FinishOpOnCompletion(&completion, PendingOp::kSendInitialMetadata);
|
2818
|
-
});
|
2819
|
-
spawner.Spawn(
|
2820
|
-
"client_promise",
|
2821
|
-
[this, client_initial_metadata = std::move(client_initial_metadata),
|
2822
|
-
token = std::move(token)]() mutable {
|
2823
|
-
return Race(
|
2824
|
-
cancel_error_.Wait(),
|
2825
|
-
Map(channel()->channel_stack()->MakeClientCallPromise(CallArgs{
|
2826
|
-
std::move(client_initial_metadata), std::move(token),
|
2827
|
-
&polling_entity_, &server_initial_metadata_.sender,
|
2828
|
-
&client_to_server_messages_.receiver,
|
2829
|
-
&server_to_client_messages_.sender}),
|
2830
|
-
[this](ServerMetadataHandle trailing_metadata) {
|
2831
|
-
// If we're cancelled the transport doesn't get to return
|
2832
|
-
// stats.
|
2833
|
-
AcceptTransportStatsFromContext();
|
2834
|
-
return trailing_metadata;
|
2835
|
-
}));
|
2836
|
-
},
|
2837
|
-
[this](ServerMetadataHandle trailing_metadata) {
|
2838
|
-
Finish(std::move(trailing_metadata));
|
2839
|
-
});
|
2840
|
-
}
|
2841
|
-
|
2842
|
-
grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
2843
|
-
size_t nops) const {
|
2844
|
-
BitSet<8> got_ops;
|
2845
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2846
|
-
const grpc_op& op = ops[op_idx];
|
2847
|
-
switch (op.op) {
|
2848
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
2849
|
-
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
2850
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2851
|
-
}
|
2852
|
-
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
2853
|
-
op.data.send_initial_metadata.metadata)) {
|
2854
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
2855
|
-
}
|
2856
|
-
break;
|
2857
|
-
case GRPC_OP_SEND_MESSAGE:
|
2858
|
-
if (!AreWriteFlagsValid(op.flags)) {
|
2859
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2860
|
-
}
|
2861
|
-
break;
|
2862
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
2863
|
-
case GRPC_OP_RECV_MESSAGE:
|
2864
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2865
|
-
break;
|
2866
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2867
|
-
if (scheduled_send_close_) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2868
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2869
|
-
break;
|
2870
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
2871
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2872
|
-
if (scheduled_receive_status_) {
|
2873
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2874
|
-
}
|
2875
|
-
break;
|
2876
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2877
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2878
|
-
return GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
2879
|
-
}
|
2880
|
-
if (got_ops.is_set(op.op)) {
|
2881
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2882
|
-
}
|
2883
|
-
got_ops.set(op.op);
|
2884
|
-
}
|
2885
|
-
return GRPC_CALL_OK;
|
2886
|
-
}
|
2887
|
-
|
2888
|
-
void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
2889
|
-
const Completion& completion) {
|
2890
|
-
Party::BulkSpawner spawner(this);
|
2891
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2892
|
-
const grpc_op& op = ops[op_idx];
|
2893
|
-
switch (op.op) {
|
2894
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
2895
|
-
if (started_.exchange(true, std::memory_order_relaxed)) break;
|
2896
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
2897
|
-
op.data.send_initial_metadata.count,
|
2898
|
-
send_initial_metadata_.get());
|
2899
|
-
PrepareOutgoingInitialMetadata(op, *send_initial_metadata_);
|
2900
|
-
if (send_deadline() != Timestamp::InfFuture()) {
|
2901
|
-
send_initial_metadata_->Set(GrpcTimeoutMetadata(), send_deadline());
|
2902
|
-
}
|
2903
|
-
send_initial_metadata_->Set(
|
2904
|
-
WaitForReady(),
|
2905
|
-
WaitForReady::ValueType{
|
2906
|
-
(op.flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
2907
|
-
(op.flags &
|
2908
|
-
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
2909
|
-
StartPromise(std::move(send_initial_metadata_), completion, spawner);
|
2910
|
-
} break;
|
2911
|
-
case GRPC_OP_RECV_INITIAL_METADATA: {
|
2912
|
-
StartRecvInitialMetadata(
|
2913
|
-
op.data.recv_initial_metadata.recv_initial_metadata, completion,
|
2914
|
-
spawner);
|
2915
|
-
} break;
|
2916
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
2917
|
-
scheduled_receive_status_ = true;
|
2918
|
-
StartRecvStatusOnClient(completion, op.data.recv_status_on_client,
|
2919
|
-
spawner);
|
2920
|
-
} break;
|
2921
|
-
case GRPC_OP_SEND_MESSAGE:
|
2922
|
-
StartSendMessage(op, completion, &client_to_server_messages_.sender,
|
2923
|
-
spawner);
|
2924
|
-
break;
|
2925
|
-
case GRPC_OP_RECV_MESSAGE:
|
2926
|
-
StartRecvMessage(
|
2927
|
-
op, completion,
|
2928
|
-
[this]() {
|
2929
|
-
return Race(server_initial_metadata_.receiver.AwaitClosed(),
|
2930
|
-
server_to_client_messages_.receiver.AwaitClosed());
|
2931
|
-
},
|
2932
|
-
&server_to_client_messages_.receiver, false, spawner);
|
2933
|
-
break;
|
2934
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2935
|
-
scheduled_send_close_ = true;
|
2936
|
-
spawner.Spawn(
|
2937
|
-
"send_close_from_client",
|
2938
|
-
[this]() {
|
2939
|
-
client_to_server_messages_.sender.Close();
|
2940
|
-
return Empty{};
|
2941
|
-
},
|
2942
|
-
[this,
|
2943
|
-
completion = AddOpToCompletion(
|
2944
|
-
completion, PendingOp::kSendCloseFromClient)](Empty) mutable {
|
2945
|
-
FinishOpOnCompletion(&completion,
|
2946
|
-
PendingOp::kSendCloseFromClient);
|
2947
|
-
});
|
2948
|
-
break;
|
2949
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2950
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2951
|
-
abort(); // unreachable
|
2952
|
-
}
|
2953
|
-
}
|
2954
|
-
}
|
2955
|
-
|
2956
|
-
grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
2957
|
-
size_t nops,
|
2958
|
-
void* notify_tag,
|
2959
|
-
bool is_notify_tag_closure) {
|
2960
|
-
if (nops == 0) {
|
2961
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
2962
|
-
return GRPC_CALL_OK;
|
2963
|
-
}
|
2964
|
-
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
2965
|
-
if (validation_result != GRPC_CALL_OK) {
|
2966
|
-
return validation_result;
|
2967
|
-
}
|
2968
|
-
Completion completion =
|
2969
|
-
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
2970
|
-
CommitBatch(ops, nops, completion);
|
2971
|
-
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
2972
|
-
return GRPC_CALL_OK;
|
2973
|
-
}
|
2974
|
-
|
2975
|
-
void ClientPromiseBasedCall::StartRecvInitialMetadata(
|
2976
|
-
grpc_metadata_array* array, const Completion& completion,
|
2977
|
-
Party::BulkSpawner& spawner) {
|
2978
|
-
spawner.Spawn(
|
2979
|
-
"recv_initial_metadata",
|
2980
|
-
[this]() {
|
2981
|
-
return Race(server_initial_metadata_.receiver.Next(),
|
2982
|
-
Map(finished(), [](Empty) {
|
2983
|
-
return NextResult<ServerMetadataHandle>(true);
|
2984
|
-
}));
|
2985
|
-
},
|
2986
|
-
[this, array,
|
2987
|
-
completion =
|
2988
|
-
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata)](
|
2989
|
-
NextResult<ServerMetadataHandle> next_metadata) mutable {
|
2990
|
-
server_initial_metadata_.sender.Close();
|
2991
|
-
ServerMetadataHandle metadata;
|
2992
|
-
if (grpc_call_trace.enabled()) {
|
2993
|
-
gpr_log(GPR_INFO, "%s[call] RecvTrailingMetadata: %s",
|
2994
|
-
DebugTag().c_str(),
|
2995
|
-
next_metadata.has_value()
|
2996
|
-
? next_metadata.value()->DebugString().c_str()
|
2997
|
-
: "null");
|
2998
|
-
}
|
2999
|
-
if (next_metadata.has_value()) {
|
3000
|
-
metadata = std::move(next_metadata.value());
|
3001
|
-
is_trailers_only_ = metadata->get(GrpcTrailersOnly()).value_or(false);
|
3002
|
-
} else {
|
3003
|
-
is_trailers_only_ = true;
|
3004
|
-
metadata = arena()->MakePooled<ServerMetadata>();
|
3005
|
-
}
|
3006
|
-
ProcessIncomingInitialMetadata(*metadata);
|
3007
|
-
PublishMetadataArray(metadata.get(), array, true);
|
3008
|
-
recv_initial_metadata_ = std::move(metadata);
|
3009
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveInitialMetadata);
|
3010
|
-
});
|
3011
|
-
}
|
3012
|
-
|
3013
|
-
void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
3014
|
-
if (grpc_call_trace.enabled()) {
|
3015
|
-
gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
|
3016
|
-
trailing_metadata->DebugString().c_str());
|
3017
|
-
}
|
3018
|
-
ResetDeadline();
|
3019
|
-
set_completed();
|
3020
|
-
client_to_server_messages_.sender.CloseWithError();
|
3021
|
-
client_to_server_messages_.receiver.CloseWithError();
|
3022
|
-
if (trailing_metadata->get(GrpcCallWasCancelled()).value_or(false)) {
|
3023
|
-
server_to_client_messages_.receiver.CloseWithError();
|
3024
|
-
server_initial_metadata_.receiver.CloseWithError();
|
3025
|
-
}
|
3026
|
-
if (auto* channelz_channel = channel()->channelz_node()) {
|
3027
|
-
if (trailing_metadata->get(GrpcStatusMetadata())
|
3028
|
-
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3029
|
-
channelz_channel->RecordCallSucceeded();
|
3030
|
-
} else {
|
3031
|
-
channelz_channel->RecordCallFailed();
|
3032
|
-
}
|
3033
|
-
}
|
3034
|
-
server_trailing_metadata_.Set(std::move(trailing_metadata));
|
3035
|
-
}
|
3036
|
-
|
3037
|
-
namespace {
|
3038
|
-
std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
|
3039
|
-
std::string out = absl::StrCat(
|
3040
|
-
trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
|
3041
|
-
? "Error received from peer"
|
3042
|
-
: "Error generated by client",
|
3043
|
-
"grpc_status: ",
|
3044
|
-
grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
|
3045
|
-
.value_or(GRPC_STATUS_UNKNOWN)));
|
3046
|
-
if (const Slice* message =
|
3047
|
-
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3048
|
-
absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
|
3049
|
-
}
|
3050
|
-
if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
|
3051
|
-
absl::StrAppend(&out, "\nStatus Context:");
|
3052
|
-
for (const std::string& annotation : *annotations) {
|
3053
|
-
absl::StrAppend(&out, "\n ", annotation);
|
3054
|
-
}
|
3055
|
-
}
|
3056
|
-
return out;
|
3057
|
-
}
|
3058
|
-
} // namespace
|
3059
|
-
|
3060
|
-
void ClientPromiseBasedCall::StartRecvStatusOnClient(
|
3061
|
-
const Completion& completion,
|
3062
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
3063
|
-
Party::BulkSpawner& spawner) {
|
3064
|
-
ForceCompletionSuccess(completion);
|
3065
|
-
spawner.Spawn(
|
3066
|
-
"recv_status_on_client", server_trailing_metadata_.Wait(),
|
3067
|
-
[this, op_args,
|
3068
|
-
completion =
|
3069
|
-
AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient)](
|
3070
|
-
ServerMetadataHandle trailing_metadata) mutable {
|
3071
|
-
const grpc_status_code status =
|
3072
|
-
trailing_metadata->get(GrpcStatusMetadata())
|
3073
|
-
.value_or(GRPC_STATUS_UNKNOWN);
|
3074
|
-
*op_args.status = status;
|
3075
|
-
Slice message_slice;
|
3076
|
-
if (Slice* message =
|
3077
|
-
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3078
|
-
message_slice = message->Ref();
|
3079
|
-
}
|
3080
|
-
SetFinalizationStatus(status, message_slice.Ref());
|
3081
|
-
*op_args.status_details = message_slice.TakeCSlice();
|
3082
|
-
if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
|
3083
|
-
*op_args.error_string =
|
3084
|
-
gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
|
3085
|
-
}
|
3086
|
-
PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata,
|
3087
|
-
true);
|
3088
|
-
recv_trailing_metadata_ = std::move(trailing_metadata);
|
3089
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveStatusOnClient);
|
3090
|
-
});
|
3091
|
-
}
|
3092
|
-
#endif
|
3093
|
-
|
3094
|
-
///////////////////////////////////////////////////////////////////////////////
|
3095
|
-
// ServerPromiseBasedCall
|
3096
|
-
|
3097
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3098
|
-
|
3099
|
-
class ServerPromiseBasedCall final : public PromiseBasedCall,
|
3100
|
-
public ServerCallContext {
|
3101
|
-
public:
|
3102
|
-
ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
|
3103
|
-
|
3104
|
-
void OrphanCall() override {}
|
3105
|
-
void CancelWithError(grpc_error_handle) override;
|
3106
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3107
|
-
bool is_notify_tag_closure) override;
|
3108
|
-
bool is_trailers_only() const override {
|
3109
|
-
Crash("is_trailers_only not implemented for server calls");
|
3110
|
-
}
|
3111
|
-
absl::string_view GetServerAuthority() const override {
|
3112
|
-
const Slice* authority_metadata =
|
3113
|
-
client_initial_metadata_->get_pointer(HttpAuthorityMetadata());
|
3114
|
-
if (authority_metadata == nullptr) return "";
|
3115
|
-
return authority_metadata->as_string_view();
|
3116
|
-
}
|
3117
|
-
|
3118
|
-
// Polling order for the server promise stack:
|
3119
|
-
//
|
3120
|
-
// │ ┌───────────────────────────────────────┐
|
3121
|
-
// │ │ ServerPromiseBasedCall ├──► Lifetime management
|
3122
|
-
// │ ├───────────────────────────────────────┤
|
3123
|
-
// │ │ ConnectedChannel ├─┐
|
3124
|
-
// │ ├───────────────────────────────────────┤ └► Interactions with the
|
3125
|
-
// │ │ ... closest to transport filter │ transport - send/recv msgs
|
3126
|
-
// │ ├───────────────────────────────────────┤ and metadata, call phase
|
3127
|
-
// │ │ ... │ ordering
|
3128
|
-
// │ ├───────────────────────────────────────┤
|
3129
|
-
// │ │ ... closest to app filter │ ┌► Request matching, initial
|
3130
|
-
// │ ├───────────────────────────────────────┤ │ setup, publishing call to
|
3131
|
-
// │ │ Server::ChannelData::MakeCallPromise ├─┘ application
|
3132
|
-
// │ ├───────────────────────────────────────┤
|
3133
|
-
// │ │ MakeTopOfServerCallPromise ├──► Send trailing metadata
|
3134
|
-
// ▼ └───────────────────────────────────────┘
|
3135
|
-
// Polling &
|
3136
|
-
// instantiation
|
3137
|
-
// order
|
3138
|
-
|
3139
|
-
std::string DebugTag() const override {
|
3140
|
-
return absl::StrFormat("SERVER_CALL[%p]: ", this);
|
3141
|
-
}
|
3142
|
-
|
3143
|
-
ServerCallContext* server_call_context() override { return this; }
|
3144
|
-
|
3145
|
-
const void* server_stream_data() override { return server_transport_data_; }
|
3146
|
-
void PublishInitialMetadata(
|
3147
|
-
ClientMetadataHandle metadata,
|
3148
|
-
grpc_metadata_array* publish_initial_metadata) override;
|
3149
|
-
ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
|
3150
|
-
CallArgs call_args, grpc_completion_queue* cq,
|
3151
|
-
absl::FunctionRef<void(grpc_call* call)> publish) override;
|
3152
|
-
|
3153
|
-
private:
|
3154
|
-
class RecvCloseOpCancelState {
|
3155
|
-
public:
|
3156
|
-
// Request that receiver be filled in per
|
3157
|
-
// grpc_op_recv_close_on_server. Returns true if the request can
|
3158
|
-
// be fulfilled immediately. Returns false if the request will be
|
3159
|
-
// fulfilled later.
|
3160
|
-
bool ReceiveCloseOnServerOpStarted(int* receiver) {
|
3161
|
-
uintptr_t state = state_.load(std::memory_order_acquire);
|
3162
|
-
uintptr_t new_state;
|
3163
|
-
do {
|
3164
|
-
switch (state) {
|
3165
|
-
case kUnset:
|
3166
|
-
new_state = reinterpret_cast<uintptr_t>(receiver);
|
3167
|
-
break;
|
3168
|
-
case kFinishedWithFailure:
|
3169
|
-
*receiver = 1;
|
3170
|
-
return true;
|
3171
|
-
case kFinishedWithSuccess:
|
3172
|
-
*receiver = 0;
|
3173
|
-
return true;
|
3174
|
-
default:
|
3175
|
-
Crash("Two threads offered ReceiveCloseOnServerOpStarted");
|
3176
|
-
}
|
3177
|
-
} while (!state_.compare_exchange_weak(state, new_state,
|
3178
|
-
std::memory_order_acq_rel,
|
3179
|
-
std::memory_order_acquire));
|
3180
|
-
return false;
|
3181
|
-
}
|
3182
|
-
|
3183
|
-
// Mark the call as having completed.
|
3184
|
-
// Returns true if this finishes a previous
|
3185
|
-
// RequestReceiveCloseOnServer.
|
3186
|
-
bool CompleteCallWithCancelledSetTo(bool cancelled) {
|
3187
|
-
uintptr_t state = state_.load(std::memory_order_acquire);
|
3188
|
-
uintptr_t new_state;
|
3189
|
-
bool r;
|
3190
|
-
do {
|
3191
|
-
switch (state) {
|
3192
|
-
case kUnset:
|
3193
|
-
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3194
|
-
r = false;
|
3195
|
-
break;
|
3196
|
-
case kFinishedWithFailure:
|
3197
|
-
return false;
|
3198
|
-
case kFinishedWithSuccess:
|
3199
|
-
Crash("unreachable");
|
3200
|
-
default:
|
3201
|
-
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3202
|
-
r = true;
|
3203
|
-
}
|
3204
|
-
} while (!state_.compare_exchange_weak(state, new_state,
|
3205
|
-
std::memory_order_acq_rel,
|
3206
|
-
std::memory_order_acquire));
|
3207
|
-
if (r) *reinterpret_cast<int*>(state) = cancelled ? 1 : 0;
|
3208
|
-
return r;
|
3209
|
-
}
|
3210
|
-
|
3211
|
-
std::string ToString() const {
|
3212
|
-
auto state = state_.load(std::memory_order_relaxed);
|
3213
|
-
switch (state) {
|
3214
|
-
case kUnset:
|
3215
|
-
return "Unset";
|
3216
|
-
case kFinishedWithFailure:
|
3217
|
-
return "FinishedWithFailure";
|
3218
|
-
case kFinishedWithSuccess:
|
3219
|
-
return "FinishedWithSuccess";
|
3220
|
-
default:
|
3221
|
-
return absl::StrFormat("WaitingForReceiver(%p)",
|
3222
|
-
reinterpret_cast<void*>(state));
|
3223
|
-
}
|
3224
|
-
}
|
3225
|
-
|
3226
|
-
private:
|
3227
|
-
static constexpr uintptr_t kUnset = 0;
|
3228
|
-
static constexpr uintptr_t kFinishedWithFailure = 1;
|
3229
|
-
static constexpr uintptr_t kFinishedWithSuccess = 2;
|
3230
|
-
// Holds one of kUnset, kFinishedWithFailure, or
|
3231
|
-
// kFinishedWithSuccess OR an int* that wants to receive the
|
3232
|
-
// final status.
|
3233
|
-
std::atomic<uintptr_t> state_{kUnset};
|
3234
|
-
};
|
3235
|
-
|
3236
|
-
void CommitBatch(const grpc_op* ops, size_t nops,
|
3237
|
-
const Completion& completion);
|
3238
|
-
void Finish(ServerMetadataHandle result);
|
3239
|
-
|
3240
|
-
ServerInterface* const server_;
|
3241
|
-
const void* const server_transport_data_;
|
3242
|
-
PipeSender<ServerMetadataHandle>* server_initial_metadata_ = nullptr;
|
3243
|
-
PipeSender<MessageHandle>* server_to_client_messages_ = nullptr;
|
3244
|
-
PipeReceiver<MessageHandle>* client_to_server_messages_ = nullptr;
|
3245
|
-
Latch<ServerMetadataHandle> send_trailing_metadata_;
|
3246
|
-
RecvCloseOpCancelState recv_close_op_cancel_state_;
|
3247
|
-
ClientMetadataHandle client_initial_metadata_;
|
3248
|
-
Completion recv_close_completion_;
|
3249
|
-
std::atomic<bool> cancelled_{false};
|
3250
|
-
};
|
3251
|
-
|
3252
|
-
ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
3253
|
-
grpc_call_create_args* args)
|
3254
|
-
: PromiseBasedCall(arena, 0, *args),
|
3255
|
-
server_(args->server),
|
3256
|
-
server_transport_data_(args->server_transport_data) {
|
3257
|
-
global_stats().IncrementServerCallsCreated();
|
3258
|
-
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3259
|
-
if (channelz_node != nullptr) {
|
3260
|
-
channelz_node->RecordCallStarted();
|
3261
|
-
}
|
3262
|
-
ScopedContext activity_context(this);
|
3263
|
-
// TODO(yashykt): In the future, we want to also enable stats and trace
|
3264
|
-
// collecting from when the call is created at the transport. The idea is that
|
3265
|
-
// the transport would create the call tracer and pass it in as part of the
|
3266
|
-
// metadata.
|
3267
|
-
// TODO(yijiem): OpenCensus and internal Census is still using this way to
|
3268
|
-
// set server call tracer. We need to refactor them to stats plugins
|
3269
|
-
// (including removing the client channel filters).
|
3270
|
-
if (args->server != nullptr &&
|
3271
|
-
args->server->server_call_tracer_factory() != nullptr) {
|
3272
|
-
auto* server_call_tracer =
|
3273
|
-
args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
|
3274
|
-
arena, args->server->channel_args());
|
3275
|
-
if (server_call_tracer != nullptr) {
|
3276
|
-
// Note that we are setting both
|
3277
|
-
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
3278
|
-
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
3279
|
-
// promise-based world, we would just a single tracer object for each
|
3280
|
-
// stack (call, subchannel_call, server_call.)
|
3281
|
-
ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
3282
|
-
server_call_tracer, nullptr);
|
3283
|
-
ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
3284
|
-
}
|
3285
|
-
}
|
3286
|
-
args->channel->channel_stack()->stats_plugin_group->AddServerCallTracers(
|
3287
|
-
context());
|
3288
|
-
Spawn("server_promise",
|
3289
|
-
channel()->channel_stack()->MakeServerCallPromise(
|
3290
|
-
CallArgs{nullptr, ClientInitialMetadataOutstandingToken::Empty(),
|
3291
|
-
nullptr, nullptr, nullptr, nullptr}),
|
3292
|
-
[this](ServerMetadataHandle result) { Finish(std::move(result)); });
|
3293
|
-
}
|
3294
|
-
|
3295
|
-
void ServerPromiseBasedCall::Finish(ServerMetadataHandle result) {
|
3296
|
-
if (grpc_call_trace.enabled()) {
|
3297
|
-
gpr_log(GPR_INFO, "%s[call] Finish: recv_close_state:%s result:%s",
|
3298
|
-
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3299
|
-
result->DebugString().c_str());
|
3300
|
-
}
|
3301
|
-
const auto status =
|
3302
|
-
result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN);
|
3303
|
-
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3304
|
-
if (channelz_node != nullptr) {
|
3305
|
-
if (status == GRPC_STATUS_OK) {
|
3306
|
-
channelz_node->RecordCallSucceeded();
|
3307
|
-
} else {
|
3308
|
-
channelz_node->RecordCallFailed();
|
3309
|
-
}
|
3310
|
-
}
|
3311
|
-
bool was_cancelled = result->get(GrpcCallWasCancelled()).value_or(true);
|
3312
|
-
if (recv_close_op_cancel_state_.CompleteCallWithCancelledSetTo(
|
3313
|
-
was_cancelled)) {
|
3314
|
-
FinishOpOnCompletion(&recv_close_completion_,
|
3315
|
-
PendingOp::kReceiveCloseOnServer);
|
3316
|
-
}
|
3317
|
-
if (was_cancelled) set_failed_before_recv_message();
|
3318
|
-
if (server_initial_metadata_ != nullptr) {
|
3319
|
-
server_initial_metadata_->Close();
|
3320
|
-
}
|
3321
|
-
Slice message_slice;
|
3322
|
-
if (Slice* message = result->get_pointer(GrpcMessageMetadata())) {
|
3323
|
-
message_slice = message->Ref();
|
3324
|
-
}
|
3325
|
-
AcceptTransportStatsFromContext();
|
3326
|
-
SetFinalizationStatus(status, std::move(message_slice));
|
3327
|
-
set_completed();
|
3328
|
-
ResetDeadline();
|
3329
|
-
PropagateCancellationToChildren();
|
3330
|
-
}
|
3331
|
-
|
3332
|
-
grpc_call_error ValidateServerBatch(const grpc_op* ops, size_t nops) {
|
3333
|
-
BitSet<8> got_ops;
|
3334
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3335
|
-
const grpc_op& op = ops[op_idx];
|
3336
|
-
switch (op.op) {
|
3337
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
3338
|
-
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
3339
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3340
|
-
}
|
3341
|
-
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
3342
|
-
op.data.send_initial_metadata.metadata)) {
|
3343
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3344
|
-
}
|
3345
|
-
break;
|
3346
|
-
case GRPC_OP_SEND_MESSAGE:
|
3347
|
-
if (!AreWriteFlagsValid(op.flags)) {
|
3348
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3349
|
-
}
|
3350
|
-
break;
|
3351
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3352
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3353
|
-
if (!ValidateMetadata(
|
3354
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3355
|
-
op.data.send_status_from_server.trailing_metadata)) {
|
3356
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3357
|
-
}
|
3358
|
-
break;
|
3359
|
-
case GRPC_OP_RECV_MESSAGE:
|
3360
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3361
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3362
|
-
break;
|
3363
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3364
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3365
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3366
|
-
return GRPC_CALL_ERROR_NOT_ON_SERVER;
|
3367
|
-
}
|
3368
|
-
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
3369
|
-
got_ops.set(op.op);
|
3370
|
-
}
|
3371
|
-
return GRPC_CALL_OK;
|
3372
|
-
}
|
3373
|
-
|
3374
|
-
void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
3375
|
-
const Completion& completion) {
|
3376
|
-
Party::BulkSpawner spawner(this);
|
3377
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3378
|
-
const grpc_op& op = ops[op_idx];
|
3379
|
-
switch (op.op) {
|
3380
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
3381
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3382
|
-
PrepareOutgoingInitialMetadata(op, *metadata);
|
3383
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
3384
|
-
op.data.send_initial_metadata.count, metadata.get());
|
3385
|
-
if (grpc_call_trace.enabled()) {
|
3386
|
-
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3387
|
-
DebugTag().c_str());
|
3388
|
-
}
|
3389
|
-
QueueSend();
|
3390
|
-
spawner.Spawn(
|
3391
|
-
"call_send_initial_metadata",
|
3392
|
-
[this, metadata = std::move(metadata)]() mutable {
|
3393
|
-
EnactSend();
|
3394
|
-
return server_initial_metadata_->Push(std::move(metadata));
|
3395
|
-
},
|
3396
|
-
[this,
|
3397
|
-
completion = AddOpToCompletion(
|
3398
|
-
completion, PendingOp::kSendInitialMetadata)](bool r) mutable {
|
3399
|
-
if (!r) {
|
3400
|
-
set_failed_before_recv_message();
|
3401
|
-
FailCompletion(completion);
|
3402
|
-
}
|
3403
|
-
FinishOpOnCompletion(&completion,
|
3404
|
-
PendingOp::kSendInitialMetadata);
|
3405
|
-
});
|
3406
|
-
} break;
|
3407
|
-
case GRPC_OP_SEND_MESSAGE:
|
3408
|
-
StartSendMessage(op, completion, server_to_client_messages_, spawner);
|
3409
|
-
break;
|
3410
|
-
case GRPC_OP_RECV_MESSAGE:
|
3411
|
-
if (cancelled_.load(std::memory_order_relaxed)) {
|
3412
|
-
set_failed_before_recv_message();
|
3413
|
-
FailCompletion(completion);
|
3414
|
-
break;
|
3415
|
-
}
|
3416
|
-
StartRecvMessage(
|
3417
|
-
op, completion, []() { return []() { return Empty{}; }; },
|
3418
|
-
client_to_server_messages_, true, spawner);
|
3419
|
-
break;
|
3420
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
3421
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3422
|
-
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3423
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3424
|
-
metadata.get());
|
3425
|
-
metadata->Set(GrpcStatusMetadata(),
|
3426
|
-
op.data.send_status_from_server.status);
|
3427
|
-
if (auto* details = op.data.send_status_from_server.status_details) {
|
3428
|
-
// TODO(ctiller): this should not be a copy, but we have callers that
|
3429
|
-
// allocate and pass in a slice created with
|
3430
|
-
// grpc_slice_from_static_string and then delete the string after
|
3431
|
-
// passing it in, which shouldn't be a supported API.
|
3432
|
-
metadata->Set(GrpcMessageMetadata(),
|
3433
|
-
Slice(grpc_slice_copy(*details)));
|
3434
|
-
}
|
3435
|
-
spawner.Spawn(
|
3436
|
-
"call_send_status_from_server",
|
3437
|
-
[this, metadata = std::move(metadata)]() mutable {
|
3438
|
-
bool r = true;
|
3439
|
-
if (send_trailing_metadata_.is_set()) {
|
3440
|
-
r = false;
|
3441
|
-
} else {
|
3442
|
-
send_trailing_metadata_.Set(std::move(metadata));
|
3443
|
-
}
|
3444
|
-
return Map(WaitForSendingStarted(), [this, r](Empty) {
|
3445
|
-
server_initial_metadata_->Close();
|
3446
|
-
server_to_client_messages_->Close();
|
3447
|
-
return r;
|
3448
|
-
});
|
3449
|
-
},
|
3450
|
-
[this, completion = AddOpToCompletion(
|
3451
|
-
completion, PendingOp::kSendStatusFromServer)](
|
3452
|
-
bool ok) mutable {
|
3453
|
-
if (!ok) {
|
3454
|
-
set_failed_before_recv_message();
|
3455
|
-
FailCompletion(completion);
|
3456
|
-
}
|
3457
|
-
FinishOpOnCompletion(&completion,
|
3458
|
-
PendingOp::kSendStatusFromServer);
|
3459
|
-
});
|
3460
|
-
} break;
|
3461
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3462
|
-
if (grpc_call_trace.enabled()) {
|
3463
|
-
gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
|
3464
|
-
DebugTag().c_str(),
|
3465
|
-
recv_close_op_cancel_state_.ToString().c_str());
|
3466
|
-
}
|
3467
|
-
ForceCompletionSuccess(completion);
|
3468
|
-
recv_close_completion_ =
|
3469
|
-
AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
|
3470
|
-
if (recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
|
3471
|
-
op.data.recv_close_on_server.cancelled)) {
|
3472
|
-
FinishOpOnCompletion(&recv_close_completion_,
|
3473
|
-
PendingOp::kReceiveCloseOnServer);
|
3474
|
-
}
|
3475
|
-
break;
|
3476
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3477
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3478
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3479
|
-
abort(); // unreachable
|
3480
|
-
}
|
3481
|
-
}
|
3482
|
-
}
|
3483
|
-
|
3484
|
-
grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
3485
|
-
size_t nops,
|
3486
|
-
void* notify_tag,
|
3487
|
-
bool is_notify_tag_closure) {
|
3488
|
-
if (nops == 0) {
|
3489
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3490
|
-
return GRPC_CALL_OK;
|
3491
|
-
}
|
3492
|
-
const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
|
3493
|
-
if (validation_result != GRPC_CALL_OK) {
|
3494
|
-
return validation_result;
|
3495
|
-
}
|
3496
|
-
Completion completion =
|
3497
|
-
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
3498
|
-
CommitBatch(ops, nops, completion);
|
3499
|
-
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
3500
|
-
return GRPC_CALL_OK;
|
3501
|
-
}
|
3502
|
-
|
3503
|
-
void ServerPromiseBasedCall::CancelWithError(absl::Status error) {
|
3504
|
-
cancelled_.store(true, std::memory_order_relaxed);
|
3505
|
-
Spawn(
|
3506
|
-
"cancel_with_error",
|
3507
|
-
[this, error = std::move(error)]() {
|
3508
|
-
if (!send_trailing_metadata_.is_set()) {
|
3509
|
-
auto md = ServerMetadataFromStatus(error);
|
3510
|
-
md->Set(GrpcCallWasCancelled(), true);
|
3511
|
-
send_trailing_metadata_.Set(std::move(md));
|
3512
|
-
}
|
3513
|
-
if (server_to_client_messages_ != nullptr) {
|
3514
|
-
server_to_client_messages_->Close();
|
3515
|
-
}
|
3516
|
-
if (server_initial_metadata_ != nullptr) {
|
3517
|
-
server_initial_metadata_->Close();
|
3518
|
-
}
|
3519
|
-
return Empty{};
|
3520
|
-
},
|
3521
|
-
[](Empty) {});
|
3522
|
-
}
|
3523
|
-
#endif
|
3524
|
-
|
3525
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3526
|
-
void ServerPromiseBasedCall::PublishInitialMetadata(
|
3527
|
-
ClientMetadataHandle metadata,
|
3528
|
-
grpc_metadata_array* publish_initial_metadata) {
|
3529
|
-
if (grpc_call_trace.enabled()) {
|
3530
|
-
gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
|
3531
|
-
metadata->DebugString().c_str());
|
3532
|
-
}
|
3533
|
-
PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
|
3534
|
-
client_initial_metadata_ = std::move(metadata);
|
3535
|
-
}
|
3536
|
-
|
3537
|
-
ArenaPromise<ServerMetadataHandle>
|
3538
|
-
ServerPromiseBasedCall::MakeTopOfServerCallPromise(
|
3539
|
-
CallArgs call_args, grpc_completion_queue* cq,
|
3540
|
-
absl::FunctionRef<void(grpc_call* call)> publish) {
|
3541
|
-
SetCompletionQueue(cq);
|
3542
|
-
call_args.polling_entity->Set(
|
3543
|
-
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)));
|
3544
|
-
server_to_client_messages_ = call_args.server_to_client_messages;
|
3545
|
-
client_to_server_messages_ = call_args.client_to_server_messages;
|
3546
|
-
server_initial_metadata_ = call_args.server_initial_metadata;
|
3547
|
-
absl::optional<Timestamp> deadline =
|
3548
|
-
client_initial_metadata_->get(GrpcTimeoutMetadata());
|
3549
|
-
if (deadline.has_value()) {
|
3550
|
-
set_send_deadline(*deadline);
|
3551
|
-
UpdateDeadline(*deadline);
|
3552
|
-
}
|
3553
|
-
ProcessIncomingInitialMetadata(*client_initial_metadata_);
|
3554
|
-
ExternalRef();
|
3555
|
-
publish(c_ptr());
|
3556
|
-
return Seq(server_to_client_messages_->AwaitClosed(),
|
3557
|
-
send_trailing_metadata_.Wait());
|
3558
|
-
}
|
3559
|
-
|
3560
|
-
///////////////////////////////////////////////////////////////////////////////
|
3561
|
-
// CallSpine based Server Call
|
3562
|
-
|
3563
|
-
class ServerCallSpine final : public PipeBasedCallSpine,
|
3564
|
-
public ServerCallContext,
|
3565
|
-
public BasicPromiseBasedCall {
|
3566
|
-
public:
|
3567
|
-
ServerCallSpine(ClientMetadataHandle client_initial_metadata,
|
3568
|
-
ServerInterface* server, Channel* channel, Arena* arena);
|
3569
|
-
|
3570
|
-
// CallSpineInterface
|
3571
|
-
Pipe<ClientMetadataHandle>& client_initial_metadata() override {
|
3572
|
-
return client_initial_metadata_;
|
3573
|
-
}
|
3574
|
-
Pipe<ServerMetadataHandle>& server_initial_metadata() override {
|
3575
|
-
return server_initial_metadata_;
|
3576
|
-
}
|
3577
|
-
Pipe<MessageHandle>& client_to_server_messages() override {
|
3578
|
-
return client_to_server_messages_;
|
3579
|
-
}
|
3580
|
-
Pipe<MessageHandle>& server_to_client_messages() override {
|
3581
|
-
return server_to_client_messages_;
|
3582
|
-
}
|
3583
|
-
Latch<ServerMetadataHandle>& cancel_latch() override { return cancel_latch_; }
|
3584
|
-
Latch<bool>& was_cancelled_latch() override { return was_cancelled_latch_; }
|
3585
|
-
Party& party() override { return *this; }
|
3586
|
-
Arena* arena() override { return BasicPromiseBasedCall::arena(); }
|
3587
|
-
void IncrementRefCount() override { InternalRef("CallSpine"); }
|
3588
|
-
void Unref() override { InternalUnref("CallSpine"); }
|
3589
|
-
|
3590
|
-
// PromiseBasedCall
|
3591
|
-
void OrphanCall() override {
|
3592
|
-
ResetDeadline();
|
3593
|
-
CancelWithError(absl::CancelledError());
|
3594
|
-
}
|
3595
|
-
void CancelWithError(grpc_error_handle error) override {
|
3596
|
-
SpawnInfallible("CancelWithError", [this, error = std::move(error)] {
|
3597
|
-
auto status = ServerMetadataFromStatus(error);
|
3598
|
-
status->Set(GrpcCallWasCancelled(), true);
|
3599
|
-
PushServerTrailingMetadata(std::move(status));
|
3600
|
-
return Empty{};
|
3601
|
-
});
|
3602
|
-
}
|
3603
|
-
bool is_trailers_only() const override {
|
3604
|
-
Crash("is_trailers_only not implemented for server calls");
|
3605
|
-
}
|
3606
|
-
absl::string_view GetServerAuthority() const override {
|
3607
|
-
Crash("unimplemented");
|
3608
|
-
}
|
3609
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3610
|
-
bool is_notify_tag_closure) override;
|
3611
|
-
|
3612
|
-
bool Completed() final { Crash("unimplemented"); }
|
3613
|
-
bool failed_before_recv_message() const final { Crash("unimplemented"); }
|
3614
|
-
|
3615
|
-
ServerCallContext* server_call_context() override { return this; }
|
3616
|
-
const void* server_stream_data() override { Crash("unimplemented"); }
|
3617
|
-
void PublishInitialMetadata(
|
3618
|
-
ClientMetadataHandle metadata,
|
3619
|
-
grpc_metadata_array* publish_initial_metadata) override;
|
3620
|
-
ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
|
3621
|
-
CallArgs, grpc_completion_queue*,
|
3622
|
-
absl::FunctionRef<void(grpc_call* call)>) override {
|
3623
|
-
Crash("unimplemented");
|
3624
|
-
}
|
3625
|
-
|
3626
|
-
void V2HackToStartCallWithoutACallFilterStack() override {}
|
3627
|
-
|
3628
|
-
ClientMetadata& UnprocessedClientInitialMetadata() override {
|
3629
|
-
Crash("not for v2");
|
3630
|
-
}
|
3631
|
-
|
3632
|
-
bool RunParty() override {
|
3633
|
-
ScopedContext ctx(this);
|
3634
|
-
return Party::RunParty();
|
3635
|
-
}
|
3636
|
-
|
3637
|
-
private:
|
3638
|
-
void CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3639
|
-
bool is_notify_tag_closure);
|
3640
|
-
StatusFlag FinishRecvMessage(NextResult<MessageHandle> result);
|
3641
|
-
|
3642
|
-
std::string DebugTag() const override {
|
3643
|
-
return absl::StrFormat("SERVER_CALL_SPINE[%p]: ", this);
|
3644
|
-
}
|
3645
|
-
|
3646
|
-
// Initial metadata from client to server
|
3647
|
-
Pipe<ClientMetadataHandle> client_initial_metadata_;
|
3648
|
-
// Initial metadata from server to client
|
3649
|
-
Pipe<ServerMetadataHandle> server_initial_metadata_;
|
3650
|
-
// Messages travelling from the application to the transport.
|
3651
|
-
Pipe<MessageHandle> client_to_server_messages_;
|
3652
|
-
// Messages travelling from the transport to the application.
|
3653
|
-
Pipe<MessageHandle> server_to_client_messages_;
|
3654
|
-
// Latch that can be set to terminate the call
|
3655
|
-
Latch<ServerMetadataHandle> cancel_latch_;
|
3656
|
-
Latch<bool> was_cancelled_latch_;
|
3657
|
-
grpc_byte_buffer** recv_message_ = nullptr;
|
3658
|
-
ClientMetadataHandle client_initial_metadata_stored_;
|
3659
|
-
};
|
3660
|
-
|
3661
|
-
ServerCallSpine::ServerCallSpine(ClientMetadataHandle client_initial_metadata,
|
3662
|
-
ServerInterface* server, Channel* channel,
|
3663
|
-
Arena* arena)
|
3664
|
-
: BasicPromiseBasedCall(arena, 0, 1,
|
3665
|
-
[channel, server]() -> grpc_call_create_args {
|
3666
|
-
grpc_call_create_args args;
|
3667
|
-
args.channel = channel->Ref();
|
3668
|
-
args.server = server;
|
3669
|
-
args.parent = nullptr;
|
3670
|
-
args.propagation_mask = 0;
|
3671
|
-
args.cq = nullptr;
|
3672
|
-
args.pollset_set_alternative = nullptr;
|
3673
|
-
args.server_transport_data =
|
3674
|
-
&args; // Arbitrary non-null pointer
|
3675
|
-
args.send_deadline = Timestamp::InfFuture();
|
3676
|
-
return args;
|
3677
|
-
}()),
|
3678
|
-
client_initial_metadata_(arena),
|
3679
|
-
server_initial_metadata_(arena),
|
3680
|
-
client_to_server_messages_(arena),
|
3681
|
-
server_to_client_messages_(arena) {
|
3682
|
-
global_stats().IncrementServerCallsCreated();
|
3683
|
-
ScopedContext ctx(this);
|
3684
|
-
channel->channel_stack()->InitServerCallSpine(this);
|
3685
|
-
SpawnGuarded("push_client_initial_metadata",
|
3686
|
-
[this, md = std::move(client_initial_metadata)]() mutable {
|
3687
|
-
return Map(client_initial_metadata_.sender.Push(std::move(md)),
|
3688
|
-
[](bool r) { return StatusFlag(r); });
|
3689
|
-
});
|
3690
|
-
}
|
3691
|
-
|
3692
|
-
void ServerCallSpine::PublishInitialMetadata(
|
3693
|
-
ClientMetadataHandle metadata,
|
3694
|
-
grpc_metadata_array* publish_initial_metadata) {
|
3695
|
-
if (grpc_call_trace.enabled()) {
|
3696
|
-
gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
|
3697
|
-
metadata->DebugString().c_str());
|
3698
|
-
}
|
3699
|
-
PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
|
3700
|
-
client_initial_metadata_stored_ = std::move(metadata);
|
3701
|
-
}
|
3702
|
-
|
3703
|
-
grpc_call_error ServerCallSpine::StartBatch(const grpc_op* ops, size_t nops,
|
3704
|
-
void* notify_tag,
|
3705
|
-
bool is_notify_tag_closure) {
|
3706
|
-
if (nops == 0) {
|
3707
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3708
|
-
return GRPC_CALL_OK;
|
3709
|
-
}
|
3710
|
-
const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
|
3711
|
-
if (validation_result != GRPC_CALL_OK) {
|
3712
|
-
return validation_result;
|
3713
|
-
}
|
3714
|
-
CommitBatch(ops, nops, notify_tag, is_notify_tag_closure);
|
3715
|
-
return GRPC_CALL_OK;
|
3716
|
-
}
|
3717
|
-
|
3718
|
-
namespace {
|
3719
|
-
template <typename SetupFn>
|
3720
|
-
class MaybeOpImpl {
|
3721
|
-
public:
|
3722
|
-
using SetupResult = decltype(std::declval<SetupFn>()(grpc_op()));
|
3723
|
-
using PromiseFactory = promise_detail::OncePromiseFactory<void, SetupResult>;
|
3724
|
-
using Promise = typename PromiseFactory::Promise;
|
3725
|
-
struct Dismissed {};
|
3726
|
-
using State = absl::variant<Dismissed, PromiseFactory, Promise>;
|
3727
|
-
|
3728
|
-
// op_ is garbage but shouldn't be uninitialized
|
3729
|
-
MaybeOpImpl() : state_(Dismissed{}), op_(GRPC_OP_RECV_STATUS_ON_CLIENT) {}
|
3730
|
-
MaybeOpImpl(SetupResult result, grpc_op_type op)
|
3731
|
-
: state_(PromiseFactory(std::move(result))), op_(op) {}
|
3732
|
-
|
3733
|
-
MaybeOpImpl(const MaybeOpImpl&) = delete;
|
3734
|
-
MaybeOpImpl& operator=(const MaybeOpImpl&) = delete;
|
3735
|
-
MaybeOpImpl(MaybeOpImpl&& other) noexcept
|
3736
|
-
: state_(MoveState(other.state_)), op_(other.op_) {}
|
3737
|
-
MaybeOpImpl& operator=(MaybeOpImpl&& other) noexcept {
|
3738
|
-
op_ = other.op_;
|
3739
|
-
if (absl::holds_alternative<Dismissed>(state_)) {
|
3740
|
-
state_.template emplace<Dismissed>();
|
3741
|
-
return *this;
|
3742
|
-
}
|
3743
|
-
// Can't move after first poll => Promise is not an option
|
3744
|
-
state_.template emplace<PromiseFactory>(
|
3745
|
-
std::move(absl::get<PromiseFactory>(other.state_)));
|
3746
|
-
return *this;
|
3747
|
-
}
|
3748
|
-
|
3749
|
-
Poll<StatusFlag> operator()() {
|
3750
|
-
if (absl::holds_alternative<Dismissed>(state_)) return Success{};
|
3751
|
-
if (absl::holds_alternative<PromiseFactory>(state_)) {
|
3752
|
-
auto& factory = absl::get<PromiseFactory>(state_);
|
3753
|
-
auto promise = factory.Make();
|
3754
|
-
state_.template emplace<Promise>(std::move(promise));
|
3755
|
-
}
|
3756
|
-
if (grpc_call_trace.enabled()) {
|
3757
|
-
gpr_log(GPR_INFO, "%sBeginPoll %s",
|
3758
|
-
Activity::current()->DebugTag().c_str(), OpName(op_).c_str());
|
3759
|
-
}
|
3760
|
-
auto& promise = absl::get<Promise>(state_);
|
3761
|
-
auto r = poll_cast<StatusFlag>(promise());
|
3762
|
-
if (grpc_call_trace.enabled()) {
|
3763
|
-
gpr_log(GPR_INFO, "%sEndPoll %s --> %s",
|
3764
|
-
Activity::current()->DebugTag().c_str(), OpName(op_).c_str(),
|
3765
|
-
r.pending() ? "PENDING" : (r.value().ok() ? "OK" : "FAILURE"));
|
3766
|
-
}
|
3767
|
-
return r;
|
3768
|
-
}
|
3769
|
-
|
3770
|
-
private:
|
3771
|
-
GPR_NO_UNIQUE_ADDRESS State state_;
|
3772
|
-
GPR_NO_UNIQUE_ADDRESS grpc_op_type op_;
|
3773
|
-
|
3774
|
-
static std::string OpName(grpc_op_type op) {
|
3775
|
-
switch (op) {
|
3776
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
3777
|
-
return "SendInitialMetadata";
|
3778
|
-
case GRPC_OP_SEND_MESSAGE:
|
3779
|
-
return "SendMessage";
|
3780
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3781
|
-
return "SendStatusFromServer";
|
3782
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3783
|
-
return "SendCloseFromClient";
|
3784
|
-
case GRPC_OP_RECV_MESSAGE:
|
3785
|
-
return "RecvMessage";
|
3786
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3787
|
-
return "RecvCloseOnServer";
|
3788
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3789
|
-
return "RecvInitialMetadata";
|
3790
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3791
|
-
return "RecvStatusOnClient";
|
3792
|
-
}
|
3793
|
-
return absl::StrCat("UnknownOp(", op, ")");
|
3794
|
-
}
|
3795
|
-
|
3796
|
-
static State MoveState(State& state) {
|
3797
|
-
if (absl::holds_alternative<Dismissed>(state)) return Dismissed{};
|
3798
|
-
// Can't move after first poll => Promise is not an option
|
3799
|
-
return std::move(absl::get<PromiseFactory>(state));
|
3800
|
-
}
|
3801
|
-
};
|
3802
|
-
|
3803
|
-
// MaybeOp captures a fairly complicated dance we need to do for the batch API.
|
3804
|
-
// We first check if an op is included or not, and if it is, we run the setup
|
3805
|
-
// function in the context of the API call (NOT in the call party).
|
3806
|
-
// This setup function returns a promise factory which we'll then run *in* the
|
3807
|
-
// party to do initial setup, and have it return the promise that we'll
|
3808
|
-
// ultimately poll on til completion.
|
3809
|
-
// Once we express our surface API in terms of core internal types this whole
|
3810
|
-
// dance will go away.
|
3811
|
-
template <typename SetupFn>
|
3812
|
-
auto MaybeOp(const grpc_op* ops, uint8_t idx, SetupFn setup) {
|
3813
|
-
if (idx == 255) {
|
3814
|
-
return MaybeOpImpl<SetupFn>();
|
3815
|
-
} else {
|
3816
|
-
return MaybeOpImpl<SetupFn>(setup(ops[idx]), ops[idx].op);
|
3817
|
-
}
|
3818
|
-
}
|
3819
|
-
|
3820
|
-
template <typename F>
|
3821
|
-
class PollBatchLogger {
|
3822
|
-
public:
|
3823
|
-
PollBatchLogger(void* tag, F f) : tag_(tag), f_(std::move(f)) {}
|
3824
|
-
|
3825
|
-
auto operator()() {
|
3826
|
-
if (grpc_call_trace.enabled()) {
|
3827
|
-
gpr_log(GPR_INFO, "Poll batch %p", tag_);
|
3828
|
-
}
|
3829
|
-
auto r = f_();
|
3830
|
-
if (grpc_call_trace.enabled()) {
|
3831
|
-
gpr_log(GPR_INFO, "Poll batch %p --> %s", tag_, ResultString(r).c_str());
|
3832
|
-
}
|
3833
|
-
return r;
|
3834
|
-
}
|
3835
|
-
|
3836
|
-
private:
|
3837
|
-
template <typename T>
|
3838
|
-
static std::string ResultString(Poll<T> r) {
|
3839
|
-
if (r.pending()) return "PENDING";
|
3840
|
-
return ResultString(r.value());
|
3841
|
-
}
|
3842
|
-
static std::string ResultString(Empty) { return "DONE"; }
|
3843
|
-
|
3844
|
-
void* tag_;
|
3845
|
-
F f_;
|
3846
|
-
};
|
3847
|
-
|
3848
|
-
template <typename F>
|
3849
|
-
PollBatchLogger<F> LogPollBatch(void* tag, F f) {
|
3850
|
-
return PollBatchLogger<F>(tag, std::move(f));
|
3851
|
-
}
|
3852
|
-
} // namespace
|
3853
|
-
|
3854
|
-
StatusFlag ServerCallSpine::FinishRecvMessage(
|
3855
|
-
NextResult<MessageHandle> result) {
|
3856
|
-
if (result.has_value()) {
|
3857
|
-
MessageHandle& message = *result;
|
3858
|
-
NoteLastMessageFlags(message->flags());
|
3859
|
-
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
3860
|
-
(incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
3861
|
-
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
3862
|
-
nullptr, 0, incoming_compression_algorithm());
|
3863
|
-
} else {
|
3864
|
-
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
3865
|
-
}
|
3866
|
-
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
3867
|
-
&(*recv_message_)->data.raw.slice_buffer);
|
3868
|
-
if (grpc_call_trace.enabled()) {
|
3869
|
-
gpr_log(GPR_INFO,
|
3870
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3871
|
-
"finishes: received %" PRIdPTR " byte message",
|
3872
|
-
DebugTag().c_str(),
|
3873
|
-
(*recv_message_)->data.raw.slice_buffer.length);
|
3874
|
-
}
|
3875
|
-
recv_message_ = nullptr;
|
3876
|
-
return Success{};
|
3877
|
-
}
|
3878
|
-
if (result.cancelled()) {
|
3879
|
-
if (grpc_call_trace.enabled()) {
|
3880
|
-
gpr_log(GPR_INFO,
|
3881
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3882
|
-
"finishes: received end-of-stream with error",
|
3883
|
-
DebugTag().c_str());
|
3884
|
-
}
|
3885
|
-
*recv_message_ = nullptr;
|
3886
|
-
recv_message_ = nullptr;
|
3887
|
-
return Failure{};
|
3888
|
-
}
|
3889
|
-
if (grpc_call_trace.enabled()) {
|
3890
|
-
gpr_log(GPR_INFO,
|
3891
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3892
|
-
"finishes: received end-of-stream",
|
3893
|
-
DebugTag().c_str());
|
3894
|
-
}
|
3895
|
-
*recv_message_ = nullptr;
|
3896
|
-
recv_message_ = nullptr;
|
3897
|
-
return Success{};
|
3898
|
-
}
|
3899
|
-
|
3900
|
-
void ServerCallSpine::CommitBatch(const grpc_op* ops, size_t nops,
|
3901
|
-
void* notify_tag,
|
3902
|
-
bool is_notify_tag_closure) {
|
3903
|
-
std::array<uint8_t, 8> got_ops{255, 255, 255, 255, 255, 255, 255, 255};
|
3904
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3905
|
-
const grpc_op& op = ops[op_idx];
|
3906
|
-
got_ops[op.op] = op_idx;
|
3907
|
-
}
|
3908
|
-
if (!is_notify_tag_closure) grpc_cq_begin_op(cq(), notify_tag);
|
3909
|
-
auto send_initial_metadata = MaybeOp(
|
3910
|
-
ops, got_ops[GRPC_OP_SEND_INITIAL_METADATA], [this](const grpc_op& op) {
|
3911
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3912
|
-
PrepareOutgoingInitialMetadata(op, *metadata);
|
3913
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
3914
|
-
op.data.send_initial_metadata.count, metadata.get());
|
3915
|
-
if (grpc_call_trace.enabled()) {
|
3916
|
-
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3917
|
-
DebugTag().c_str());
|
3918
|
-
}
|
3919
|
-
return [this, metadata = std::move(metadata)]() mutable {
|
3920
|
-
return Map(server_initial_metadata_.sender.Push(std::move(metadata)),
|
3921
|
-
[this](bool r) {
|
3922
|
-
server_initial_metadata_.sender.Close();
|
3923
|
-
return StatusFlag(r);
|
3924
|
-
});
|
3925
|
-
};
|
3926
|
-
});
|
3927
|
-
auto send_message =
|
3928
|
-
MaybeOp(ops, got_ops[GRPC_OP_SEND_MESSAGE], [this](const grpc_op& op) {
|
3929
|
-
SliceBuffer send;
|
3930
|
-
grpc_slice_buffer_swap(
|
3931
|
-
&op.data.send_message.send_message->data.raw.slice_buffer,
|
3932
|
-
send.c_slice_buffer());
|
3933
|
-
auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
|
3934
|
-
return [this, msg = std::move(msg)]() mutable {
|
3935
|
-
return Map(server_to_client_messages_.sender.Push(std::move(msg)),
|
3936
|
-
[](bool r) { return StatusFlag(r); });
|
3937
|
-
};
|
3938
|
-
});
|
3939
|
-
auto send_trailing_metadata = MaybeOp(
|
3940
|
-
ops, got_ops[GRPC_OP_SEND_STATUS_FROM_SERVER], [this](const grpc_op& op) {
|
3941
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3942
|
-
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3943
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3944
|
-
metadata.get());
|
3945
|
-
metadata->Set(GrpcStatusMetadata(),
|
3946
|
-
op.data.send_status_from_server.status);
|
3947
|
-
if (auto* details = op.data.send_status_from_server.status_details) {
|
3948
|
-
// TODO(ctiller): this should not be a copy, but we have
|
3949
|
-
// callers that allocate and pass in a slice created with
|
3950
|
-
// grpc_slice_from_static_string and then delete the string
|
3951
|
-
// after passing it in, which shouldn't be a supported API.
|
3952
|
-
metadata->Set(GrpcMessageMetadata(),
|
3953
|
-
Slice(grpc_slice_copy(*details)));
|
3954
|
-
}
|
3955
|
-
CHECK(metadata != nullptr);
|
3956
|
-
return [this, metadata = std::move(metadata)]() mutable {
|
3957
|
-
CHECK(metadata != nullptr);
|
3958
|
-
return [this,
|
3959
|
-
metadata = std::move(metadata)]() mutable -> Poll<Success> {
|
3960
|
-
CHECK(metadata != nullptr);
|
3961
|
-
PushServerTrailingMetadata(std::move(metadata));
|
3962
|
-
return Success{};
|
3963
|
-
};
|
3964
|
-
};
|
3965
|
-
});
|
3966
|
-
auto recv_message =
|
3967
|
-
MaybeOp(ops, got_ops[GRPC_OP_RECV_MESSAGE], [this](const grpc_op& op) {
|
3968
|
-
CHECK_EQ(recv_message_, nullptr);
|
3969
|
-
recv_message_ = op.data.recv_message.recv_message;
|
3970
|
-
return [this]() mutable {
|
3971
|
-
return Map(client_to_server_messages_.receiver.Next(),
|
3972
|
-
[this](NextResult<MessageHandle> msg) {
|
3973
|
-
return FinishRecvMessage(std::move(msg));
|
3974
|
-
});
|
3975
|
-
};
|
3976
|
-
});
|
3977
|
-
auto primary_ops = AllOk<StatusFlag>(
|
3978
|
-
TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
|
3979
|
-
std::move(send_message)),
|
3980
|
-
std::move(send_trailing_metadata)),
|
3981
|
-
std::move(recv_message));
|
3982
|
-
if (got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER] != 255) {
|
3983
|
-
auto recv_trailing_metadata = MaybeOp(
|
3984
|
-
ops, got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER], [this](const grpc_op& op) {
|
3985
|
-
return [this, cancelled = op.data.recv_close_on_server.cancelled]() {
|
3986
|
-
return Map(WasCancelled(),
|
3987
|
-
[cancelled, this](bool result) -> Success {
|
3988
|
-
ResetDeadline();
|
3989
|
-
*cancelled = result ? 1 : 0;
|
3990
|
-
return Success{};
|
3991
|
-
});
|
3992
|
-
};
|
3993
|
-
});
|
3994
|
-
SpawnInfallible(
|
3995
|
-
"final-batch",
|
3996
|
-
[primary_ops = std::move(primary_ops),
|
3997
|
-
recv_trailing_metadata = std::move(recv_trailing_metadata),
|
3998
|
-
is_notify_tag_closure, notify_tag, this]() mutable {
|
3999
|
-
return LogPollBatch(
|
4000
|
-
notify_tag,
|
4001
|
-
Seq(std::move(primary_ops), std::move(recv_trailing_metadata),
|
4002
|
-
[is_notify_tag_closure, notify_tag, this](StatusFlag) {
|
4003
|
-
return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
|
4004
|
-
absl::OkStatus(), cq());
|
4005
|
-
}));
|
4006
|
-
});
|
4007
|
-
} else {
|
4008
|
-
SpawnInfallible("batch", [primary_ops = std::move(primary_ops),
|
4009
|
-
is_notify_tag_closure, notify_tag,
|
4010
|
-
this]() mutable {
|
4011
|
-
return LogPollBatch(
|
4012
|
-
notify_tag,
|
4013
|
-
Seq(std::move(primary_ops),
|
4014
|
-
[is_notify_tag_closure, notify_tag, this](StatusFlag r) {
|
4015
|
-
return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
|
4016
|
-
StatusCast<grpc_error_handle>(r), cq());
|
4017
|
-
}));
|
4018
|
-
});
|
4019
|
-
}
|
4020
|
-
}
|
4021
|
-
|
4022
|
-
RefCountedPtr<CallSpineInterface> MakeServerCall(
|
4023
|
-
ClientMetadataHandle client_initial_metadata, ServerInterface* server,
|
4024
|
-
Channel* channel, Arena* arena) {
|
4025
|
-
return RefCountedPtr<ServerCallSpine>(arena->New<ServerCallSpine>(
|
4026
|
-
std::move(client_initial_metadata), server, channel, arena));
|
4027
|
-
}
|
4028
|
-
#else
|
4029
|
-
RefCountedPtr<CallSpineInterface> MakeServerCall(ClientMetadataHandle,
|
4030
|
-
ServerInterface*, Channel*,
|
4031
|
-
Arena*) {
|
4032
|
-
Crash("not implemented");
|
4033
|
-
}
|
4034
|
-
#endif
|
4035
|
-
|
4036
|
-
} // namespace grpc_core
|
4037
|
-
|
4038
|
-
///////////////////////////////////////////////////////////////////////////////
|
4039
|
-
// C-based API
|
4040
|
-
|
4041
|
-
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
4042
|
-
grpc_core::ExecCtx exec_ctx;
|
4043
|
-
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
4044
|
-
}
|
4045
|
-
|
4046
|
-
size_t grpc_call_get_initial_size_estimate() {
|
4047
|
-
return grpc_core::FilterStackCall::InitialSizeEstimate();
|
4048
|
-
}
|
4049
|
-
|
4050
|
-
grpc_error_handle grpc_call_create(grpc_call_create_args* args,
|
4051
|
-
grpc_call** out_call) {
|
4052
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
4053
|
-
if (grpc_core::IsPromiseBasedClientCallEnabled() &&
|
4054
|
-
args->server_transport_data == nullptr && args->channel->is_promising()) {
|
4055
|
-
return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
|
4056
|
-
args, out_call);
|
4057
|
-
}
|
4058
|
-
#endif
|
4059
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
4060
|
-
if (grpc_core::IsPromiseBasedServerCallEnabled() &&
|
4061
|
-
args->server_transport_data != nullptr && args->channel->is_promising()) {
|
4062
|
-
return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
|
4063
|
-
args, out_call);
|
4064
|
-
}
|
4065
|
-
#endif
|
4066
|
-
return grpc_core::FilterStackCall::Create(args, out_call);
|
4067
|
-
}
|
4068
|
-
|
4069
|
-
void grpc_call_set_completion_queue(grpc_call* call,
|
4070
|
-
grpc_completion_queue* cq) {
|
4071
|
-
grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
|
4072
|
-
}
|
4073
|
-
|
4074
|
-
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
4075
|
-
|
4076
|
-
void grpc_call_unref(grpc_call* c) {
|
4077
|
-
grpc_core::ExecCtx exec_ctx;
|
4078
|
-
grpc_core::Call::FromC(c)->ExternalUnref();
|
401
|
+
void grpc_call_unref(grpc_call* c) {
|
402
|
+
grpc_core::ExecCtx exec_ctx;
|
403
|
+
grpc_core::Call::FromC(c)->ExternalUnref();
|
4079
404
|
}
|
4080
405
|
|
4081
406
|
char* grpc_call_get_peer(grpc_call* call) {
|
4082
407
|
return grpc_core::Call::FromC(call)->GetPeer();
|
4083
408
|
}
|
4084
409
|
|
4085
|
-
grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element) {
|
4086
|
-
return grpc_core::FilterStackCall::FromTopElem(surface_element)->c_ptr();
|
4087
|
-
}
|
4088
|
-
|
4089
410
|
grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
|
4090
411
|
GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved));
|
4091
412
|
CHECK_EQ(reserved, nullptr);
|
@@ -4122,7 +443,7 @@ void grpc_call_cancel_internal(grpc_call* call) {
|
|
4122
443
|
|
4123
444
|
grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
|
4124
445
|
grpc_call* call) {
|
4125
|
-
return grpc_core::Call::FromC(call)->
|
446
|
+
return grpc_core::Call::FromC(call)->incoming_compression_algorithm();
|
4126
447
|
}
|
4127
448
|
|
4128
449
|
uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
|
@@ -4166,13 +487,17 @@ grpc_call_error grpc_call_start_batch_and_execute(grpc_call* call,
|
|
4166
487
|
return grpc_core::Call::FromC(call)->StartBatch(ops, nops, closure, true);
|
4167
488
|
}
|
4168
489
|
|
4169
|
-
void
|
4170
|
-
|
4171
|
-
|
490
|
+
void grpc_call_tracer_set(grpc_call* call,
|
491
|
+
grpc_core::ClientCallTracer* tracer) {
|
492
|
+
grpc_core::Arena* arena = grpc_call_get_arena(call);
|
493
|
+
return arena->SetContext<grpc_core::CallTracerAnnotationInterface>(tracer);
|
4172
494
|
}
|
4173
495
|
|
4174
|
-
void*
|
4175
|
-
|
496
|
+
void* grpc_call_tracer_get(grpc_call* call) {
|
497
|
+
grpc_core::Arena* arena = grpc_call_get_arena(call);
|
498
|
+
auto* call_tracer =
|
499
|
+
arena->GetContext<grpc_core::CallTracerAnnotationInterface>();
|
500
|
+
return call_tracer;
|
4176
501
|
}
|
4177
502
|
|
4178
503
|
uint8_t grpc_call_is_client(grpc_call* call) {
|