grpc 1.64.3 → 1.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Makefile +57 -58
- data/include/grpc/event_engine/event_engine.h +13 -6
- data/include/grpc/impl/channel_arg_names.h +7 -3
- data/include/grpc/module.modulemap +1 -0
- data/include/grpc/passive_listener.h +62 -0
- data/include/grpc/support/log.h +7 -17
- data/include/grpc/support/port_platform.h +3 -0
- data/src/core/channelz/channel_trace.cc +1 -1
- data/src/core/channelz/channel_trace.h +1 -1
- data/src/core/channelz/channelz.cc +3 -3
- data/src/core/channelz/channelz.h +7 -7
- data/src/core/channelz/channelz_registry.cc +4 -3
- data/src/core/client_channel/backup_poller.cc +4 -5
- data/src/core/client_channel/client_channel.cc +1324 -0
- data/src/core/client_channel/client_channel.h +243 -0
- data/src/core/client_channel/client_channel_filter.cc +266 -709
- data/src/core/client_channel/client_channel_filter.h +11 -64
- data/src/core/client_channel/client_channel_internal.h +16 -5
- data/src/core/client_channel/client_channel_plugin.cc +1 -14
- data/src/core/client_channel/client_channel_service_config.h +3 -3
- data/src/core/client_channel/config_selector.cc +1 -1
- data/src/core/client_channel/config_selector.h +1 -1
- data/src/core/client_channel/dynamic_filters.cc +3 -3
- data/src/core/client_channel/dynamic_filters.h +1 -3
- data/src/core/client_channel/load_balanced_call_destination.cc +336 -0
- data/src/core/client_channel/load_balanced_call_destination.h +49 -0
- data/src/core/client_channel/retry_filter.cc +2 -9
- data/src/core/client_channel/retry_filter.h +2 -7
- data/src/core/client_channel/retry_filter_legacy_call_data.cc +65 -72
- data/src/core/client_channel/retry_filter_legacy_call_data.h +0 -2
- data/src/core/client_channel/retry_service_config.cc +4 -5
- data/src/core/client_channel/retry_service_config.h +3 -3
- data/src/core/client_channel/subchannel.cc +220 -112
- data/src/core/client_channel/subchannel.h +31 -18
- data/src/core/client_channel/subchannel_pool_interface.cc +0 -2
- data/src/core/client_channel/subchannel_pool_interface.h +2 -4
- data/src/core/client_channel/subchannel_stream_client.cc +36 -49
- data/src/core/client_channel/subchannel_stream_client.h +2 -4
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +7 -10
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +1 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +7 -0
- data/src/core/ext/filters/census/grpc_context.cc +2 -4
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +8 -15
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h +2 -0
- data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +3 -7
- data/src/core/ext/filters/fault_injection/fault_injection_filter.h +1 -0
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h +3 -3
- data/src/core/ext/filters/http/client/http_client_filter.cc +1 -0
- data/src/core/ext/filters/http/client/http_client_filter.h +1 -0
- data/src/core/ext/filters/http/client_authority_filter.cc +1 -0
- data/src/core/ext/filters/http/client_authority_filter.h +1 -0
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +10 -15
- data/src/core/ext/filters/http/message_compress/compression_filter.h +2 -0
- data/src/core/ext/filters/http/server/http_server_filter.cc +2 -2
- data/src/core/ext/filters/http/server/http_server_filter.h +1 -0
- data/src/core/ext/filters/message_size/message_size_filter.cc +6 -9
- data/src/core/ext/filters/message_size/message_size_filter.h +6 -6
- data/src/core/ext/filters/rbac/rbac_filter.cc +2 -5
- data/src/core/ext/filters/rbac/rbac_filter.h +1 -0
- data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +2 -2
- data/src/core/ext/filters/rbac/rbac_service_config_parser.h +1 -1
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +2 -6
- data/src/core/ext/filters/stateful_session/stateful_session_filter.h +1 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +3 -3
- data/src/core/ext/transport/chttp2/alpn/alpn.cc +1 -1
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +8 -25
- data/src/core/ext/transport/chttp2/client/chttp2_connector.h +0 -5
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +223 -148
- data/src/core/ext/transport/chttp2/server/chttp2_server.h +33 -0
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +131 -107
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +8 -8
- data/src/core/ext/transport/chttp2/transport/context_list_entry.h +1 -1
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +2 -4
- data/src/core/ext/transport/chttp2/transport/flow_control.h +0 -2
- data/src/core/ext/transport/chttp2/transport/frame_ping.cc +4 -6
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +1 -2
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +9 -5
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +5 -4
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +9 -2
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +32 -43
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +1 -3
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +3 -8
- data/src/core/ext/transport/chttp2/transport/http2_settings.cc +1 -1
- data/src/core/ext/transport/chttp2/transport/http2_settings.h +1 -1
- data/src/core/ext/transport/chttp2/transport/internal.h +29 -19
- data/src/core/ext/transport/chttp2/transport/parsing.cc +15 -25
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.cc +0 -2
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.h +0 -2
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc +29 -13
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.h +5 -4
- data/src/core/ext/transport/chttp2/transport/stream_lists.cc +3 -5
- data/src/core/ext/transport/chttp2/transport/writing.cc +24 -25
- data/src/core/ext/transport/inproc/inproc_transport.cc +56 -32
- data/src/core/ext/transport/inproc/inproc_transport.h +1 -3
- data/src/core/ext/transport/inproc/legacy_inproc_transport.cc +13 -15
- data/src/core/ext/transport/inproc/legacy_inproc_transport.h +0 -2
- data/src/core/handshaker/handshaker.cc +6 -14
- data/src/core/handshaker/http_connect/http_connect_handshaker.cc +9 -17
- data/src/core/handshaker/http_connect/http_proxy_mapper.cc +3 -2
- data/src/core/handshaker/security/secure_endpoint.cc +38 -32
- data/src/core/handshaker/security/secure_endpoint.h +0 -2
- data/src/core/handshaker/security/security_handshaker.cc +25 -37
- data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc +2 -1
- data/src/core/lib/address_utils/parse_address.cc +27 -39
- data/src/core/lib/address_utils/sockaddr_utils.cc +5 -6
- data/src/core/lib/avl/avl.h +1 -1
- data/src/core/lib/channel/channel_args.cc +13 -17
- data/src/core/lib/channel/channel_args.h +19 -8
- data/src/core/lib/channel/channel_stack.cc +5 -63
- data/src/core/lib/channel/channel_stack.h +13 -37
- data/src/core/lib/channel/channel_stack_builder.h +0 -5
- data/src/core/lib/channel/channel_stack_builder_impl.cc +0 -142
- data/src/core/lib/channel/channel_stack_builder_impl.h +0 -2
- data/src/core/lib/channel/connected_channel.cc +37 -676
- data/src/core/lib/channel/promise_based_filter.cc +41 -47
- data/src/core/lib/channel/promise_based_filter.h +124 -477
- data/src/core/lib/channel/status_util.cc +1 -1
- data/src/core/lib/compression/compression.cc +1 -1
- data/src/core/lib/compression/message_compress.cc +6 -6
- data/src/core/lib/config/config_vars.cc +3 -8
- data/src/core/lib/config/config_vars.h +1 -5
- data/src/core/lib/debug/event_log.h +1 -1
- data/src/core/lib/debug/trace.cc +43 -59
- data/src/core/lib/debug/trace.h +2 -97
- data/src/core/lib/debug/trace_flags.cc +255 -0
- data/src/core/lib/debug/trace_flags.h +133 -0
- data/src/core/lib/debug/trace_impl.h +115 -0
- data/src/core/lib/event_engine/ares_resolver.cc +5 -7
- data/src/core/lib/event_engine/ares_resolver.h +1 -3
- data/src/core/lib/event_engine/cf_engine/cf_engine.cc +1 -1
- data/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc +17 -22
- data/src/core/lib/event_engine/event_engine.cc +29 -4
- data/src/core/lib/event_engine/extensions/supports_fd.h +7 -0
- data/src/core/lib/event_engine/extensions/tcp_trace.h +43 -0
- data/src/core/lib/event_engine/forkable.cc +4 -5
- data/src/core/lib/event_engine/forkable.h +0 -11
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +10 -11
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +4 -3
- data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc +1 -1
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +19 -33
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +3 -2
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +24 -7
- data/src/core/lib/event_engine/posix_engine/posix_engine.h +2 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +14 -16
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +18 -22
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +13 -17
- data/src/core/lib/event_engine/posix_engine/timer.cc +1 -1
- data/src/core/lib/event_engine/posix_engine/timer_manager.cc +4 -6
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +2 -1
- data/src/core/lib/event_engine/shim.cc +1 -1
- data/src/core/lib/event_engine/tcp_socket_utils.cc +6 -8
- data/src/core/lib/event_engine/thread_local.h +1 -1
- data/src/core/lib/event_engine/thread_pool/thread_count.h +1 -1
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc +19 -21
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h +3 -6
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc +14 -13
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h +4 -3
- data/src/core/lib/event_engine/trace.h +6 -17
- data/src/core/lib/event_engine/windows/iocp.h +1 -1
- data/src/core/lib/event_engine/windows/win_socket.cc +23 -17
- data/src/core/lib/event_engine/windows/win_socket.h +4 -5
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +6 -9
- data/src/core/lib/event_engine/windows/windows_engine.cc +201 -87
- data/src/core/lib/event_engine/windows/windows_engine.h +136 -25
- data/src/core/lib/event_engine/windows/windows_listener.cc +12 -23
- data/src/core/lib/experiments/experiments.cc +35 -151
- data/src/core/lib/experiments/experiments.h +12 -45
- data/src/core/lib/gprpp/bitset.h +1 -1
- data/src/core/lib/gprpp/crash.cc +2 -3
- data/src/core/lib/gprpp/dual_ref_counted.h +45 -33
- data/src/core/lib/gprpp/dump_args.cc +54 -0
- data/src/core/lib/gprpp/dump_args.h +69 -0
- data/src/core/lib/gprpp/glob.cc +70 -0
- data/src/core/lib/gprpp/glob.h +29 -0
- data/src/core/lib/gprpp/per_cpu.cc +1 -1
- data/src/core/lib/gprpp/posix/stat.cc +3 -4
- data/src/core/lib/gprpp/posix/thd.cc +8 -9
- data/src/core/lib/gprpp/ref_counted.h +30 -22
- data/src/core/lib/gprpp/single_set_ptr.h +5 -3
- data/src/core/lib/gprpp/status_helper.cc +11 -30
- data/src/core/lib/gprpp/status_helper.h +3 -31
- data/src/core/lib/gprpp/time.cc +3 -4
- data/src/core/lib/gprpp/time.h +3 -2
- data/src/core/lib/gprpp/unique_type_name.h +1 -1
- data/src/core/lib/gprpp/validation_errors.cc +10 -1
- data/src/core/lib/gprpp/validation_errors.h +11 -0
- data/src/core/lib/gprpp/windows/stat.cc +3 -4
- data/src/core/lib/gprpp/windows/thd.cc +3 -2
- data/src/core/lib/gprpp/work_serializer.cc +48 -57
- data/src/core/lib/iomgr/buffer_list.cc +4 -2
- data/src/core/lib/iomgr/call_combiner.cc +18 -27
- data/src/core/lib/iomgr/call_combiner.h +1 -3
- data/src/core/lib/iomgr/cfstream_handle.cc +4 -6
- data/src/core/lib/iomgr/closure.h +2 -4
- data/src/core/lib/iomgr/combiner.cc +6 -8
- data/src/core/lib/iomgr/combiner.h +0 -2
- data/src/core/lib/iomgr/endpoint.cc +0 -6
- data/src/core/lib/iomgr/endpoint.h +0 -2
- data/src/core/lib/iomgr/endpoint_cfstream.cc +19 -41
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +1 -1
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +5 -6
- data/src/core/lib/iomgr/error.cc +13 -21
- data/src/core/lib/iomgr/error.h +1 -1
- data/src/core/lib/iomgr/ev_apple.cc +3 -5
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +43 -42
- data/src/core/lib/iomgr/ev_poll_posix.cc +38 -29
- data/src/core/lib/iomgr/ev_posix.cc +8 -9
- data/src/core/lib/iomgr/ev_posix.h +10 -7
- data/src/core/lib/iomgr/event_engine_shims/closure.cc +2 -2
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +14 -28
- data/src/core/lib/iomgr/exec_ctx.cc +2 -2
- data/src/core/lib/iomgr/exec_ctx.h +1 -1
- data/src/core/lib/iomgr/executor.cc +6 -15
- data/src/core/lib/iomgr/executor.h +1 -1
- data/src/core/lib/iomgr/fork_posix.cc +8 -10
- data/src/core/lib/iomgr/fork_windows.cc +3 -1
- data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +2 -3
- data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +3 -5
- data/src/core/lib/iomgr/internal_errqueue.cc +4 -2
- data/src/core/lib/iomgr/iocp_windows.cc +4 -3
- data/src/core/lib/iomgr/iomgr.cc +13 -17
- data/src/core/lib/iomgr/lockfree_event.cc +3 -5
- data/src/core/lib/iomgr/pollset.h +0 -2
- data/src/core/lib/iomgr/pollset_windows.cc +0 -2
- data/src/core/lib/iomgr/resolve_address_posix.cc +7 -14
- data/src/core/lib/iomgr/resolve_address_windows.cc +1 -1
- data/src/core/lib/iomgr/socket_factory_posix.cc +1 -1
- data/src/core/lib/iomgr/socket_mutator.cc +1 -1
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +11 -17
- data/src/core/lib/iomgr/socket_windows.cc +4 -6
- data/src/core/lib/iomgr/tcp_client_cfstream.cc +3 -5
- data/src/core/lib/iomgr/tcp_client_posix.cc +9 -15
- data/src/core/lib/iomgr/tcp_client_windows.cc +2 -4
- data/src/core/lib/iomgr/tcp_posix.cc +57 -84
- data/src/core/lib/iomgr/tcp_posix.h +0 -2
- data/src/core/lib/iomgr/tcp_server_posix.cc +3 -3
- data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +4 -6
- data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +7 -7
- data/src/core/lib/iomgr/tcp_server_windows.cc +10 -16
- data/src/core/lib/iomgr/tcp_windows.cc +25 -41
- data/src/core/lib/iomgr/timer_generic.cc +17 -20
- data/src/core/lib/iomgr/timer_heap.cc +1 -1
- data/src/core/lib/iomgr/timer_manager.cc +17 -30
- data/src/core/lib/iomgr/unix_sockets_posix.cc +1 -1
- data/src/core/lib/iomgr/vsock.cc +1 -1
- data/src/core/lib/iomgr/wakeup_fd_pipe.cc +3 -3
- data/src/core/lib/promise/activity.h +27 -4
- data/src/core/lib/promise/cancel_callback.h +24 -0
- data/src/core/lib/promise/context.h +11 -0
- data/src/core/lib/promise/detail/basic_seq.h +1 -2
- data/src/core/lib/promise/detail/join_state.h +354 -398
- data/src/core/lib/promise/detail/promise_like.h +6 -5
- data/src/core/lib/promise/detail/seq_state.h +1178 -1178
- data/src/core/lib/promise/for_each.h +6 -6
- data/src/core/lib/promise/interceptor_list.h +6 -7
- data/src/core/lib/promise/latch.h +9 -9
- data/src/core/lib/promise/map.h +17 -0
- data/src/core/lib/promise/observable.h +182 -0
- data/src/core/lib/promise/party.cc +7 -8
- data/src/core/lib/promise/party.h +10 -8
- data/src/core/lib/promise/pipe.h +16 -35
- data/src/core/lib/promise/promise.h +1 -0
- data/src/core/lib/promise/status_flag.h +2 -0
- data/src/core/lib/resource_quota/arena.cc +56 -79
- data/src/core/lib/resource_quota/arena.h +118 -209
- data/src/core/lib/resource_quota/memory_quota.cc +12 -13
- data/src/core/lib/resource_quota/memory_quota.h +2 -3
- data/src/core/lib/resource_quota/periodic_update.cc +1 -1
- data/src/core/lib/resource_quota/resource_quota.h +1 -1
- data/src/core/lib/security/authorization/authorization_policy_provider.h +1 -1
- data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +1 -1
- data/src/core/lib/security/authorization/evaluate_args.cc +6 -8
- data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +5 -6
- data/src/core/lib/security/authorization/grpc_server_authz_filter.h +1 -0
- data/src/core/lib/security/authorization/matchers.cc +3 -3
- data/src/core/lib/security/certificate_provider/certificate_provider_factory.h +2 -2
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +2 -3
- data/src/core/lib/security/context/security_context.cc +12 -13
- data/src/core/lib/security/context/security_context.h +31 -8
- data/src/core/lib/security/credentials/alts/alts_credentials.h +1 -1
- data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +3 -1
- data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +2 -3
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +5 -5
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +3 -3
- data/src/core/lib/security/credentials/call_creds_util.cc +2 -1
- data/src/core/lib/security/credentials/channel_creds_registry.h +2 -2
- data/src/core/lib/security/credentials/channel_creds_registry_init.cc +5 -3
- data/src/core/lib/security/credentials/composite/composite_credentials.h +1 -1
- data/src/core/lib/security/credentials/credentials.cc +6 -6
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +4 -4
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +2 -2
- data/src/core/lib/security/credentials/external/external_account_credentials.cc +9 -11
- data/src/core/lib/security/credentials/external/external_account_credentials.h +3 -3
- data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +2 -2
- data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +4 -4
- data/src/core/lib/security/credentials/external/url_external_account_credentials.h +1 -1
- data/src/core/lib/security/credentials/fake/fake_credentials.h +1 -1
- data/src/core/lib/security/credentials/google_default/credentials_generic.cc +3 -3
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +11 -11
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +1 -1
- data/src/core/lib/security/credentials/iam/iam_credentials.h +1 -1
- data/src/core/lib/security/credentials/jwt/json_token.cc +14 -15
- data/src/core/lib/security/credentials/jwt/json_token.h +1 -1
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +6 -5
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +1 -1
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +50 -54
- data/src/core/lib/security/credentials/jwt/jwt_verifier.h +1 -1
- data/src/core/lib/security/credentials/local/local_credentials.h +1 -1
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +12 -11
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +4 -4
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +6 -7
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +1 -3
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +15 -21
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +21 -30
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +4 -4
- data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +3 -5
- data/src/core/lib/security/credentials/tls/tls_credentials.cc +14 -16
- data/src/core/lib/security/credentials/tls/tls_utils.cc +4 -4
- data/src/core/lib/security/credentials/xds/xds_credentials.cc +1 -1
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +13 -16
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +15 -12
- data/src/core/lib/security/security_connector/load_system_roots_supported.cc +6 -6
- data/src/core/lib/security/security_connector/load_system_roots_windows.cc +1 -1
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +8 -12
- data/src/core/lib/security/security_connector/security_connector.cc +1 -4
- data/src/core/lib/security/security_connector/security_connector.h +1 -3
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +17 -19
- data/src/core/lib/security/security_connector/ssl_utils.cc +19 -21
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +29 -40
- data/src/core/lib/security/transport/auth_filters.h +1 -0
- data/src/core/lib/security/transport/client_auth_filter.cc +7 -13
- data/src/core/lib/security/transport/server_auth_filter.cc +3 -8
- data/src/core/lib/security/util/json_util.h +1 -1
- data/src/core/lib/slice/slice.h +1 -1
- data/src/core/lib/slice/slice_refcount.h +2 -4
- data/src/core/lib/slice/slice_string_helpers.cc +1 -1
- data/src/core/lib/surface/api_trace.h +1 -3
- data/src/core/lib/surface/call.cc +64 -3738
- data/src/core/lib/surface/call.h +41 -143
- data/src/core/lib/surface/call_log_batch.cc +1 -1
- data/src/core/lib/surface/call_utils.cc +276 -0
- data/src/core/lib/surface/call_utils.h +449 -0
- data/src/core/lib/surface/channel.cc +8 -3
- data/src/core/lib/surface/channel.h +10 -7
- data/src/core/lib/surface/channel_create.cc +14 -6
- data/src/core/lib/surface/channel_create.h +3 -2
- data/src/core/lib/surface/channel_init.cc +21 -77
- data/src/core/lib/surface/channel_init.h +19 -97
- data/src/core/lib/surface/client_call.cc +419 -0
- data/src/core/lib/surface/client_call.h +180 -0
- data/src/core/lib/surface/completion_queue.cc +28 -33
- data/src/core/lib/surface/completion_queue.h +0 -8
- data/src/core/lib/surface/filter_stack_call.cc +1157 -0
- data/src/core/lib/surface/filter_stack_call.h +369 -0
- data/src/core/lib/surface/init.cc +7 -6
- data/src/core/lib/surface/lame_client.cc +1 -1
- data/src/core/lib/surface/legacy_channel.cc +40 -27
- data/src/core/lib/surface/legacy_channel.h +9 -18
- data/src/core/lib/surface/server_call.cc +222 -0
- data/src/core/lib/surface/server_call.h +167 -0
- data/src/core/lib/surface/version.cc +2 -2
- data/src/core/lib/transport/bdp_estimator.cc +3 -5
- data/src/core/lib/transport/bdp_estimator.h +2 -4
- data/src/core/lib/transport/call_arena_allocator.h +9 -7
- data/src/core/lib/transport/call_destination.h +76 -0
- data/src/core/lib/transport/call_filters.cc +28 -10
- data/src/core/lib/transport/call_filters.h +128 -22
- data/src/core/lib/transport/call_spine.cc +5 -6
- data/src/core/lib/transport/call_spine.h +159 -334
- data/src/core/lib/transport/connectivity_state.cc +8 -10
- data/src/core/lib/transport/connectivity_state.h +0 -2
- data/src/core/lib/transport/interception_chain.cc +155 -0
- data/src/core/lib/transport/interception_chain.h +236 -0
- data/src/core/lib/transport/metadata_batch.h +10 -1
- data/src/core/lib/transport/metadata_info.h +1 -1
- data/src/core/lib/transport/transport.cc +3 -6
- data/src/core/lib/transport/transport.h +43 -40
- data/src/core/load_balancing/child_policy_handler.cc +8 -8
- data/src/core/load_balancing/endpoint_list.cc +5 -5
- data/src/core/load_balancing/endpoint_list.h +1 -1
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.cc +1 -0
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.h +1 -0
- data/src/core/load_balancing/grpclb/grpclb.cc +25 -29
- data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc +1 -1
- data/src/core/load_balancing/grpclb/load_balancer_api.cc +3 -4
- data/src/core/load_balancing/health_check_client.cc +10 -13
- data/src/core/load_balancing/lb_policy.cc +5 -8
- data/src/core/load_balancing/lb_policy.h +19 -3
- data/src/core/load_balancing/lb_policy_factory.h +1 -1
- data/src/core/load_balancing/lb_policy_registry.cc +2 -3
- data/src/core/load_balancing/lb_policy_registry.h +1 -1
- data/src/core/load_balancing/oob_backend_metric.cc +2 -4
- data/src/core/load_balancing/outlier_detection/outlier_detection.cc +33 -35
- data/src/core/load_balancing/outlier_detection/outlier_detection.h +3 -3
- data/src/core/load_balancing/pick_first/pick_first.cc +65 -65
- data/src/core/load_balancing/priority/priority.cc +26 -28
- data/src/core/load_balancing/ring_hash/ring_hash.cc +11 -13
- data/src/core/load_balancing/ring_hash/ring_hash.h +3 -3
- data/src/core/load_balancing/rls/rls.cc +82 -82
- data/src/core/load_balancing/round_robin/round_robin.cc +17 -20
- data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +54 -43
- data/src/core/load_balancing/weighted_target/weighted_target.cc +21 -24
- data/src/core/load_balancing/xds/cds.cc +14 -16
- data/src/core/load_balancing/xds/xds_cluster_impl.cc +16 -18
- data/src/core/load_balancing/xds/xds_cluster_manager.cc +15 -17
- data/src/core/load_balancing/xds/xds_override_host.cc +40 -41
- data/src/core/load_balancing/xds/xds_override_host.h +3 -3
- data/src/core/load_balancing/xds/xds_wrr_locality.cc +10 -12
- data/src/core/plugin_registry/grpc_plugin_registry.cc +5 -1
- data/src/core/resolver/binder/binder_resolver.cc +3 -2
- data/src/core/resolver/dns/c_ares/dns_resolver_ares.cc +3 -2
- data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +1 -1
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc +7 -14
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.h +1 -5
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +1 -1
- data/src/core/resolver/dns/dns_resolver_plugin.cc +6 -5
- data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc +4 -9
- data/src/core/resolver/dns/event_engine/service_config_helper.cc +5 -5
- data/src/core/resolver/dns/native/dns_resolver.cc +8 -9
- data/src/core/resolver/endpoint_addresses.cc +1 -1
- data/src/core/resolver/fake/fake_resolver.cc +1 -1
- data/src/core/resolver/fake/fake_resolver.h +1 -1
- data/src/core/resolver/google_c2p/google_c2p_resolver.cc +13 -14
- data/src/core/resolver/polling_resolver.cc +30 -35
- data/src/core/resolver/resolver.cc +2 -6
- data/src/core/resolver/resolver.h +0 -2
- data/src/core/resolver/resolver_registry.cc +6 -8
- data/src/core/resolver/sockaddr/sockaddr_resolver.cc +3 -3
- data/src/core/resolver/xds/xds_dependency_manager.cc +22 -23
- data/src/core/resolver/xds/xds_resolver.cc +13 -15
- data/src/core/server/server.cc +269 -389
- data/src/core/server/server.h +37 -19
- data/src/core/server/server_call_tracer_filter.cc +7 -14
- data/src/core/server/server_config_selector.h +1 -1
- data/src/core/server/server_config_selector_filter.cc +3 -3
- data/src/core/server/server_interface.h +2 -0
- data/src/core/server/xds_channel_stack_modifier.cc +1 -1
- data/src/core/server/xds_channel_stack_modifier.h +1 -1
- data/src/core/server/xds_server_config_fetcher.cc +1 -4
- data/src/core/service_config/service_config.h +1 -1
- data/src/core/service_config/service_config_call_data.h +13 -11
- data/src/core/service_config/service_config_channel_arg_filter.cc +6 -4
- data/src/core/service_config/service_config_impl.cc +5 -5
- data/src/core/service_config/service_config_impl.h +1 -1
- data/src/core/service_config/service_config_parser.cc +3 -6
- data/src/core/service_config/service_config_parser.h +1 -1
- data/src/core/{lib/channel → telemetry}/call_tracer.cc +20 -30
- data/src/core/{lib/channel → telemetry}/call_tracer.h +32 -9
- data/src/core/{lib/debug → telemetry}/histogram_view.cc +1 -1
- data/src/core/{lib/debug → telemetry}/histogram_view.h +3 -3
- data/src/core/telemetry/metrics.cc +178 -0
- data/src/core/telemetry/metrics.h +562 -0
- data/src/core/{lib/debug → telemetry}/stats.cc +1 -1
- data/src/core/{lib/debug → telemetry}/stats.h +5 -5
- data/src/core/{lib/debug → telemetry}/stats_data.cc +1 -1
- data/src/core/{lib/debug → telemetry}/stats_data.h +4 -4
- data/src/core/{lib/channel → telemetry}/tcp_tracer.h +3 -3
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +12 -13
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +25 -27
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +32 -33
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +2 -1
- data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +2 -1
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +5 -4
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +5 -3
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +1 -1
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +4 -3
- data/src/core/tsi/fake_transport_security.cc +14 -17
- data/src/core/tsi/local_transport_security.cc +6 -5
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +4 -4
- data/src/core/tsi/ssl_transport_security.cc +76 -81
- data/src/core/tsi/ssl_transport_security_utils.cc +74 -18
- data/src/core/tsi/ssl_transport_security_utils.h +11 -0
- data/src/core/tsi/transport_security.cc +0 -4
- data/src/core/tsi/transport_security.h +0 -2
- data/src/core/tsi/transport_security_interface.h +0 -4
- data/src/core/{lib/gpr → util}/alloc.h +3 -3
- data/src/core/{lib/gpr → util}/android/log.cc +0 -19
- data/src/core/{lib/gpr → util}/atm.cc +1 -1
- data/src/core/{ext/gcp/metadata_query.cc → util/gcp_metadata_query.cc} +25 -26
- data/src/core/{ext/gcp/metadata_query.h → util/gcp_metadata_query.h} +11 -11
- data/src/core/{lib/http → util/http_client}/format_request.cc +4 -3
- data/src/core/{lib/http → util/http_client}/format_request.h +6 -5
- data/src/core/{lib/http → util/http_client}/httpcli.cc +9 -10
- data/src/core/{lib/http → util/http_client}/httpcli.h +6 -5
- data/src/core/{lib/http → util/http_client}/httpcli_security_connector.cc +9 -9
- data/src/core/{lib/http → util/http_client}/httpcli_ssl_credentials.h +5 -4
- data/src/core/{lib/http → util/http_client}/parser.cc +4 -5
- data/src/core/{lib/http → util/http_client}/parser.h +5 -6
- data/src/core/{lib → util}/json/json.h +5 -4
- data/src/core/{lib → util}/json/json_args.h +5 -5
- data/src/core/{lib → util}/json/json_channel_args.h +6 -6
- data/src/core/{lib → util}/json/json_object_loader.cc +3 -2
- data/src/core/{lib → util}/json/json_object_loader.h +7 -7
- data/src/core/{lib → util}/json/json_reader.cc +3 -2
- data/src/core/{lib → util}/json/json_reader.h +6 -6
- data/src/core/{lib → util}/json/json_util.cc +4 -4
- data/src/core/{lib → util}/json/json_util.h +6 -6
- data/src/core/{lib → util}/json/json_writer.cc +3 -3
- data/src/core/{lib → util}/json/json_writer.h +6 -6
- data/src/core/{lib/gpr → util}/linux/log.cc +0 -45
- data/src/core/util/log.cc +165 -0
- data/src/core/{lib/gpr → util}/msys/tmpfile.cc +2 -2
- data/src/core/{lib/gpr → util}/posix/cpu.cc +1 -1
- data/src/core/{lib/gpr → util}/posix/log.cc +0 -42
- data/src/core/{lib/gpr → util}/posix/time.cc +1 -1
- data/src/core/{lib/gpr → util}/posix/tmpfile.cc +2 -2
- data/src/core/{lib/gpr → util}/spinlock.h +3 -3
- data/src/core/{lib/gpr → util}/string.cc +2 -2
- data/src/core/{lib/gpr → util}/string.h +3 -3
- data/src/core/{lib/gpr → util}/time_precise.cc +1 -1
- data/src/core/{lib/gpr → util}/time_precise.h +3 -3
- data/src/core/{lib/gpr → util}/tmpfile.h +3 -3
- data/src/core/{lib/gpr → util}/useful.h +3 -3
- data/src/core/{lib/gpr → util}/windows/log.cc +1 -44
- data/src/core/{lib/gpr → util}/windows/string.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/string_util.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/time.cc +1 -1
- data/src/core/{lib/gpr → util}/windows/tmpfile.cc +1 -1
- data/src/core/xds/grpc/certificate_provider_store.cc +3 -3
- data/src/core/xds/grpc/certificate_provider_store.h +4 -4
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc +3 -3
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.h +3 -3
- data/src/core/xds/grpc/xds_audit_logger_registry.h +1 -1
- data/src/core/xds/grpc/xds_bootstrap_grpc.cc +5 -5
- data/src/core/xds/grpc/xds_bootstrap_grpc.h +3 -3
- data/src/core/xds/grpc/xds_certificate_provider.h +1 -1
- data/src/core/xds/grpc/xds_client_grpc.cc +27 -23
- data/src/core/xds/grpc/xds_client_grpc.h +2 -2
- data/src/core/xds/grpc/xds_cluster.cc +4 -5
- data/src/core/xds/grpc/xds_cluster.h +1 -1
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.cc +2 -2
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.h +1 -1
- data/src/core/xds/grpc/xds_common_types.cc +1 -1
- data/src/core/xds/grpc/xds_common_types.h +1 -1
- data/src/core/xds/grpc/xds_endpoint.cc +4 -5
- data/src/core/xds/grpc/xds_http_fault_filter.cc +2 -2
- data/src/core/xds/grpc/xds_http_filters.h +2 -2
- data/src/core/xds/grpc/xds_http_rbac_filter.cc +3 -3
- data/src/core/xds/grpc/xds_http_stateful_session_filter.cc +2 -2
- data/src/core/xds/grpc/xds_lb_policy_registry.h +1 -1
- data/src/core/xds/grpc/xds_listener.cc +4 -6
- data/src/core/xds/grpc/xds_route_config.cc +7 -8
- data/src/core/xds/grpc/xds_transport_grpc.cc +2 -2
- data/src/core/xds/grpc/xds_transport_grpc.h +1 -1
- data/src/core/xds/xds_client/xds_api.cc +5 -9
- data/src/core/xds/xds_client/xds_bootstrap.cc +1 -1
- data/src/core/xds/xds_client/xds_bootstrap.h +1 -1
- data/src/core/xds/xds_client/xds_client.cc +39 -45
- data/src/core/xds/xds_client/xds_client.h +0 -3
- data/src/core/xds/xds_client/xds_client_stats.cc +6 -6
- data/src/core/xds/xds_client/xds_client_stats.h +2 -2
- data/src/ruby/bin/math_pb.rb +1 -22
- data/src/ruby/ext/grpc/rb_call.c +8 -1
- data/src/ruby/ext/grpc/rb_completion_queue.c +15 -32
- data/src/ruby/ext/grpc/rb_completion_queue.h +7 -1
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +2 -2
- data/src/ruby/ext/grpc/rb_server.c +39 -22
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/src/ruby/pb/grpc/health/v1/health_pb.rb +1 -22
- data/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb +5 -5
- data/src/ruby/pb/grpc/testing/metrics_pb.rb +10 -19
- data/src/ruby/pb/grpc/testing/metrics_services_pb.rb +5 -5
- data/src/ruby/pb/src/proto/grpc/testing/empty_pb.rb +1 -22
- data/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +1 -22
- data/src/ruby/pb/src/proto/grpc/testing/test_pb.rb +1 -22
- data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +4 -0
- data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +12 -12
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +66 -41
- data/third_party/boringssl-with-bazel/src/crypto/dilithium/dilithium.c +1497 -0
- data/third_party/boringssl-with-bazel/src/crypto/dilithium/internal.h +58 -0
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +10 -3
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +2 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +5 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +45 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +33 -23
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/fips_shared_support.c +3 -6
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +9 -4
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +6 -0
- data/third_party/boringssl-with-bazel/src/crypto/internal.h +7 -0
- data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +0 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509/v3_utl.c +49 -16
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +0 -10
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +7 -17
- data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +8 -6
- data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +3 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/experimental/dilithium.h +125 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +0 -23
- data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +1 -1
- data/third_party/boringssl-with-bazel/src/ssl/internal.h +4 -10
- metadata +103 -93
- data/src/core/ext/transport/chttp2/transport/http_trace.cc +0 -19
- data/src/core/ext/transport/chttp2/transport/http_trace.h +0 -24
- data/src/core/ext/transport/inproc/inproc_plugin.cc +0 -23
- data/src/core/handshaker/security/tsi_error.cc +0 -31
- data/src/core/handshaker/security/tsi_error.h +0 -30
- data/src/core/lib/channel/channel_stack_trace.cc +0 -19
- data/src/core/lib/channel/channel_stack_trace.h +0 -24
- data/src/core/lib/channel/context.h +0 -105
- data/src/core/lib/channel/metrics.cc +0 -334
- data/src/core/lib/channel/metrics.h +0 -365
- data/src/core/lib/event_engine/trace.cc +0 -25
- data/src/core/lib/gpr/log.cc +0 -166
- data/src/core/lib/iomgr/ev_windows.cc +0 -30
- data/src/core/lib/promise/trace.cc +0 -20
- data/src/core/lib/promise/trace.h +0 -24
- data/src/core/lib/resource_quota/trace.cc +0 -19
- data/src/core/lib/resource_quota/trace.h +0 -24
- data/src/core/lib/slice/slice_refcount.cc +0 -20
- data/src/core/lib/surface/api_trace.cc +0 -25
- data/src/core/lib/surface/call_trace.h +0 -24
- data/src/core/lib/surface/wait_for_cq_end_op.cc +0 -75
- data/src/core/lib/surface/wait_for_cq_end_op.h +0 -72
- data/src/core/lib/transport/batch_builder.cc +0 -172
- data/src/core/lib/transport/batch_builder.h +0 -474
- data/src/core/resolver/xds/xds_resolver_trace.cc +0 -25
- data/src/core/resolver/xds/xds_resolver_trace.h +0 -30
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -122
- /data/src/core/{lib/gpr → util}/alloc.cc +0 -0
- /data/src/core/{lib/gpr → util}/iphone/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/linux/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/posix/string.cc +0 -0
- /data/src/core/{lib/gpr → util}/posix/sync.cc +0 -0
- /data/src/core/{lib/gpr → util}/sync.cc +0 -0
- /data/src/core/{lib/gpr → util}/sync_abseil.cc +0 -0
- /data/src/core/{lib/gpr → util}/time.cc +0 -0
- /data/src/core/{lib/gpr → util}/windows/cpu.cc +0 -0
- /data/src/core/{lib/gpr → util}/windows/sync.cc +0 -0
@@ -25,8 +25,10 @@
|
|
25
25
|
|
26
26
|
#include <algorithm>
|
27
27
|
#include <atomic>
|
28
|
+
#include <cstdint>
|
28
29
|
#include <memory>
|
29
30
|
#include <new>
|
31
|
+
#include <queue>
|
30
32
|
#include <string>
|
31
33
|
#include <type_traits>
|
32
34
|
#include <utility>
|
@@ -34,6 +36,7 @@
|
|
34
36
|
|
35
37
|
#include "absl/base/thread_annotations.h"
|
36
38
|
#include "absl/log/check.h"
|
39
|
+
#include "absl/log/log.h"
|
37
40
|
#include "absl/status/status.h"
|
38
41
|
#include "absl/strings/str_cat.h"
|
39
42
|
#include "absl/strings/str_format.h"
|
@@ -57,21 +60,15 @@
|
|
57
60
|
|
58
61
|
#include "src/core/channelz/channelz.h"
|
59
62
|
#include "src/core/lib/channel/call_finalization.h"
|
60
|
-
#include "src/core/lib/channel/call_tracer.h"
|
61
63
|
#include "src/core/lib/channel/channel_stack.h"
|
62
|
-
#include "src/core/lib/channel/context.h"
|
63
64
|
#include "src/core/lib/channel/status_util.h"
|
64
65
|
#include "src/core/lib/compression/compression_internal.h"
|
65
|
-
#include "src/core/lib/debug/stats.h"
|
66
|
-
#include "src/core/lib/debug/stats_data.h"
|
67
66
|
#include "src/core/lib/experiments/experiments.h"
|
68
|
-
#include "src/core/lib/gpr/alloc.h"
|
69
|
-
#include "src/core/lib/gpr/time_precise.h"
|
70
|
-
#include "src/core/lib/gpr/useful.h"
|
71
67
|
#include "src/core/lib/gprpp/bitset.h"
|
72
68
|
#include "src/core/lib/gprpp/cpp_impl_of.h"
|
73
69
|
#include "src/core/lib/gprpp/crash.h"
|
74
70
|
#include "src/core/lib/gprpp/debug_location.h"
|
71
|
+
#include "src/core/lib/gprpp/match.h"
|
75
72
|
#include "src/core/lib/gprpp/ref_counted.h"
|
76
73
|
#include "src/core/lib/gprpp/ref_counted_ptr.h"
|
77
74
|
#include "src/core/lib/gprpp/status_helper.h"
|
@@ -82,10 +79,10 @@
|
|
82
79
|
#include "src/core/lib/promise/activity.h"
|
83
80
|
#include "src/core/lib/promise/all_ok.h"
|
84
81
|
#include "src/core/lib/promise/arena_promise.h"
|
82
|
+
#include "src/core/lib/promise/cancel_callback.h"
|
85
83
|
#include "src/core/lib/promise/context.h"
|
86
84
|
#include "src/core/lib/promise/latch.h"
|
87
85
|
#include "src/core/lib/promise/map.h"
|
88
|
-
#include "src/core/lib/promise/party.h"
|
89
86
|
#include "src/core/lib/promise/pipe.h"
|
90
87
|
#include "src/core/lib/promise/poll.h"
|
91
88
|
#include "src/core/lib/promise/race.h"
|
@@ -100,17 +97,17 @@
|
|
100
97
|
#include "src/core/lib/surface/channel.h"
|
101
98
|
#include "src/core/lib/surface/completion_queue.h"
|
102
99
|
#include "src/core/lib/surface/validate_metadata.h"
|
103
|
-
#include "src/core/lib/surface/wait_for_cq_end_op.h"
|
104
|
-
#include "src/core/lib/transport/batch_builder.h"
|
105
100
|
#include "src/core/lib/transport/error_utils.h"
|
101
|
+
#include "src/core/lib/transport/metadata.h"
|
106
102
|
#include "src/core/lib/transport/metadata_batch.h"
|
107
103
|
#include "src/core/lib/transport/transport.h"
|
108
104
|
#include "src/core/server/server_interface.h"
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
105
|
+
#include "src/core/telemetry/call_tracer.h"
|
106
|
+
#include "src/core/telemetry/stats.h"
|
107
|
+
#include "src/core/telemetry/stats_data.h"
|
108
|
+
#include "src/core/util/alloc.h"
|
109
|
+
#include "src/core/util/time_precise.h"
|
110
|
+
#include "src/core/util/useful.h"
|
114
111
|
|
115
112
|
namespace grpc_core {
|
116
113
|
|
@@ -121,10 +118,19 @@ using GrpcClosure = Closure;
|
|
121
118
|
///////////////////////////////////////////////////////////////////////////////
|
122
119
|
// Call
|
123
120
|
|
121
|
+
Call::Call(bool is_client, Timestamp send_deadline, RefCountedPtr<Arena> arena,
|
122
|
+
grpc_event_engine::experimental::EventEngine* event_engine)
|
123
|
+
: arena_(std::move(arena)),
|
124
|
+
send_deadline_(send_deadline),
|
125
|
+
is_client_(is_client),
|
126
|
+
event_engine_(event_engine) {
|
127
|
+
arena_->SetContext<Call>(this);
|
128
|
+
}
|
129
|
+
|
124
130
|
Call::ParentCall* Call::GetOrCreateParentCall() {
|
125
131
|
ParentCall* p = parent_call_.load(std::memory_order_acquire);
|
126
132
|
if (p == nullptr) {
|
127
|
-
p =
|
133
|
+
p = arena()->New<ParentCall>();
|
128
134
|
ParentCall* expected = nullptr;
|
129
135
|
if (!parent_call_.compare_exchange_strong(expected, p,
|
130
136
|
std::memory_order_release,
|
@@ -160,8 +166,8 @@ absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
|
|
160
166
|
"Census tracing propagation requested without Census context "
|
161
167
|
"propagation");
|
162
168
|
}
|
163
|
-
|
164
|
-
|
169
|
+
arena()->SetContext<census_context>(
|
170
|
+
parent->arena()->GetContext<census_context>());
|
165
171
|
} else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
166
172
|
return absl::UnknownError(
|
167
173
|
"Census context propagation requested without Census tracing "
|
@@ -243,28 +249,6 @@ void Call::PropagateCancellationToChildren() {
|
|
243
249
|
}
|
244
250
|
}
|
245
251
|
|
246
|
-
char* Call::GetPeer() {
|
247
|
-
Slice peer_slice = GetPeerString();
|
248
|
-
if (!peer_slice.empty()) {
|
249
|
-
absl::string_view peer_string_view = peer_slice.as_string_view();
|
250
|
-
char* peer_string =
|
251
|
-
static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
|
252
|
-
memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
|
253
|
-
peer_string[peer_string_view.size()] = '\0';
|
254
|
-
return peer_string;
|
255
|
-
}
|
256
|
-
char* peer_string = grpc_channel_get_target(channel_->c_ptr());
|
257
|
-
if (peer_string != nullptr) return peer_string;
|
258
|
-
return gpr_strdup("unknown");
|
259
|
-
}
|
260
|
-
|
261
|
-
void Call::DeleteThis() {
|
262
|
-
RefCountedPtr<Channel> channel = std::move(channel_);
|
263
|
-
Arena* arena = arena_;
|
264
|
-
this->~Call();
|
265
|
-
channel->DestroyArena(arena);
|
266
|
-
}
|
267
|
-
|
268
252
|
void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
|
269
253
|
grpc_metadata_batch& md) {
|
270
254
|
// TODO(juanlishen): If the user has already specified a compression
|
@@ -279,7 +263,7 @@ void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
|
|
279
263
|
op.data.send_initial_metadata.maybe_compression_level.level;
|
280
264
|
level_set = true;
|
281
265
|
} else {
|
282
|
-
const grpc_compression_options copts =
|
266
|
+
const grpc_compression_options copts = compression_options();
|
283
267
|
if (copts.default_level.is_set) {
|
284
268
|
level_set = true;
|
285
269
|
effective_compression_level = copts.default_level.level;
|
@@ -305,26 +289,25 @@ void Call::ProcessIncomingInitialMetadata(grpc_metadata_batch& md) {
|
|
305
289
|
Slice* peer_string = md.get_pointer(PeerString());
|
306
290
|
if (peer_string != nullptr) SetPeerString(peer_string->Ref());
|
307
291
|
|
308
|
-
|
309
|
-
md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
292
|
+
SetIncomingCompressionAlgorithm(
|
293
|
+
md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE));
|
310
294
|
encodings_accepted_by_peer_ =
|
311
295
|
md.Take(GrpcAcceptEncodingMetadata())
|
312
296
|
.value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
|
313
297
|
|
314
|
-
const grpc_compression_options
|
315
|
-
channel_->compression_options();
|
298
|
+
const grpc_compression_options copts = compression_options();
|
316
299
|
const grpc_compression_algorithm compression_algorithm =
|
317
|
-
|
318
|
-
if (GPR_UNLIKELY(
|
319
|
-
|
320
|
-
|
300
|
+
incoming_compression_algorithm();
|
301
|
+
if (GPR_UNLIKELY(
|
302
|
+
!CompressionAlgorithmSet::FromUint32(copts.enabled_algorithms_bitset)
|
303
|
+
.IsSet(compression_algorithm))) {
|
321
304
|
// check if algorithm is supported by current channel config
|
322
305
|
HandleCompressionAlgorithmDisabled(compression_algorithm);
|
323
306
|
}
|
324
307
|
// GRPC_COMPRESS_NONE is always set.
|
325
308
|
DCHECK(encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
326
309
|
if (GPR_UNLIKELY(!encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
327
|
-
if (GRPC_TRACE_FLAG_ENABLED(
|
310
|
+
if (GRPC_TRACE_FLAG_ENABLED(compression)) {
|
328
311
|
HandleCompressionAlgorithmNotAccepted(compression_algorithm);
|
329
312
|
}
|
330
313
|
}
|
@@ -347,7 +330,7 @@ void Call::HandleCompressionAlgorithmDisabled(
|
|
347
330
|
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
348
331
|
std::string error_msg =
|
349
332
|
absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
|
350
|
-
|
333
|
+
LOG(ERROR) << error_msg;
|
351
334
|
CancelWithError(grpc_error_set_int(absl::UnimplementedError(error_msg),
|
352
335
|
StatusIntProperty::kRpcStatus,
|
353
336
|
GRPC_STATUS_UNIMPLEMENTED));
|
@@ -355,7 +338,7 @@ void Call::HandleCompressionAlgorithmDisabled(
|
|
355
338
|
|
356
339
|
void Call::UpdateDeadline(Timestamp deadline) {
|
357
340
|
ReleasableMutexLock lock(&deadline_mu_);
|
358
|
-
if (
|
341
|
+
if (GRPC_TRACE_FLAG_ENABLED(call)) {
|
359
342
|
gpr_log(GPR_DEBUG, "[call %p] UpdateDeadline from=%s to=%s", this,
|
360
343
|
deadline_.ToString().c_str(), deadline.ToString().c_str());
|
361
344
|
}
|
@@ -367,22 +350,20 @@ void Call::UpdateDeadline(Timestamp deadline) {
|
|
367
350
|
StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
|
368
351
|
return;
|
369
352
|
}
|
370
|
-
auto* const event_engine = channel()->event_engine();
|
371
353
|
if (deadline_ != Timestamp::InfFuture()) {
|
372
|
-
if (!
|
354
|
+
if (!event_engine_->Cancel(deadline_task_)) return;
|
373
355
|
} else {
|
374
356
|
InternalRef("deadline");
|
375
357
|
}
|
376
358
|
deadline_ = deadline;
|
377
|
-
deadline_task_ =
|
359
|
+
deadline_task_ = event_engine_->RunAfter(deadline - Timestamp::Now(), this);
|
378
360
|
}
|
379
361
|
|
380
362
|
void Call::ResetDeadline() {
|
381
363
|
{
|
382
364
|
MutexLock lock(&deadline_mu_);
|
383
365
|
if (deadline_ == Timestamp::InfFuture()) return;
|
384
|
-
|
385
|
-
if (!event_engine->Cancel(deadline_task_)) return;
|
366
|
+
if (!event_engine_->Cancel(deadline_task_)) return;
|
386
367
|
deadline_ = Timestamp::InfFuture();
|
387
368
|
}
|
388
369
|
InternalUnref("deadline[reset]");
|
@@ -391,3700 +372,41 @@ void Call::ResetDeadline() {
|
|
391
372
|
void Call::Run() {
|
392
373
|
ApplicationCallbackExecCtx callback_exec_ctx;
|
393
374
|
ExecCtx exec_ctx;
|
375
|
+
GRPC_TRACE_LOG(call, INFO)
|
376
|
+
<< "call deadline expired "
|
377
|
+
<< GRPC_DUMP_ARGS(Timestamp::Now(), send_deadline_);
|
394
378
|
CancelWithError(grpc_error_set_int(
|
395
379
|
absl::DeadlineExceededError("Deadline Exceeded"),
|
396
380
|
StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
|
397
381
|
InternalUnref("deadline[run]");
|
398
382
|
}
|
399
383
|
|
400
|
-
|
401
|
-
// FilterStackCall
|
402
|
-
// To be removed once promise conversion is complete
|
403
|
-
|
404
|
-
class FilterStackCall final : public Call {
|
405
|
-
public:
|
406
|
-
~FilterStackCall() override {
|
407
|
-
for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
|
408
|
-
if (context_[i].destroy) {
|
409
|
-
context_[i].destroy(context_[i].value);
|
410
|
-
}
|
411
|
-
}
|
412
|
-
gpr_free(static_cast<void*>(const_cast<char*>(final_info_.error_string)));
|
413
|
-
}
|
414
|
-
|
415
|
-
bool Completed() override {
|
416
|
-
return gpr_atm_acq_load(&received_final_op_atm_) != 0;
|
417
|
-
}
|
418
|
-
|
419
|
-
// TODO(ctiller): return absl::StatusOr<SomeSmartPointer<Call>>?
|
420
|
-
static grpc_error_handle Create(grpc_call_create_args* args,
|
421
|
-
grpc_call** out_call);
|
422
|
-
|
423
|
-
static Call* FromTopElem(grpc_call_element* elem) {
|
424
|
-
return FromCallStack(grpc_call_stack_from_top_element(elem));
|
425
|
-
}
|
426
|
-
|
427
|
-
grpc_call_stack* call_stack() override {
|
428
|
-
return reinterpret_cast<grpc_call_stack*>(
|
429
|
-
reinterpret_cast<char*>(this) +
|
430
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(*this)));
|
431
|
-
}
|
432
|
-
|
433
|
-
grpc_event_engine::experimental::EventEngine* event_engine() const override {
|
434
|
-
return channel()->event_engine();
|
435
|
-
}
|
436
|
-
|
437
|
-
grpc_call_element* call_elem(size_t idx) {
|
438
|
-
return grpc_call_stack_element(call_stack(), idx);
|
439
|
-
}
|
440
|
-
|
441
|
-
CallCombiner* call_combiner() { return &call_combiner_; }
|
442
|
-
|
443
|
-
void CancelWithError(grpc_error_handle error) override;
|
444
|
-
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
445
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
446
|
-
bool is_notify_tag_closure) override;
|
447
|
-
void ExternalRef() override { ext_ref_.Ref(); }
|
448
|
-
void ExternalUnref() override;
|
449
|
-
void InternalRef(const char* reason) override {
|
450
|
-
GRPC_CALL_STACK_REF(call_stack(), reason);
|
451
|
-
}
|
452
|
-
void InternalUnref(const char* reason) override {
|
453
|
-
GRPC_CALL_STACK_UNREF(call_stack(), reason);
|
454
|
-
}
|
455
|
-
|
456
|
-
void ContextSet(grpc_context_index elem, void* value,
|
457
|
-
void (*destroy)(void* value)) override;
|
458
|
-
void* ContextGet(grpc_context_index elem) const override {
|
459
|
-
return context_[elem].value;
|
460
|
-
}
|
461
|
-
|
462
|
-
bool is_trailers_only() const override {
|
463
|
-
bool result = is_trailers_only_;
|
464
|
-
DCHECK(!result || recv_initial_metadata_.TransportSize() == 0);
|
465
|
-
return result;
|
466
|
-
}
|
467
|
-
|
468
|
-
bool failed_before_recv_message() const override {
|
469
|
-
return call_failed_before_recv_message_;
|
470
|
-
}
|
471
|
-
|
472
|
-
absl::string_view GetServerAuthority() const override {
|
473
|
-
const Slice* authority_metadata =
|
474
|
-
recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
|
475
|
-
if (authority_metadata == nullptr) return "";
|
476
|
-
return authority_metadata->as_string_view();
|
477
|
-
}
|
478
|
-
|
479
|
-
static size_t InitialSizeEstimate() {
|
480
|
-
return sizeof(FilterStackCall) +
|
481
|
-
sizeof(BatchControl) * kMaxConcurrentBatches;
|
482
|
-
}
|
483
|
-
|
484
|
-
private:
|
485
|
-
class ScopedContext : public promise_detail::Context<Arena> {
|
486
|
-
public:
|
487
|
-
explicit ScopedContext(FilterStackCall* call)
|
488
|
-
: promise_detail::Context<Arena>(call->arena()) {}
|
489
|
-
};
|
490
|
-
|
491
|
-
static constexpr gpr_atm kRecvNone = 0;
|
492
|
-
static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
|
493
|
-
|
494
|
-
enum class PendingOp {
|
495
|
-
kRecvMessage,
|
496
|
-
kRecvInitialMetadata,
|
497
|
-
kRecvTrailingMetadata,
|
498
|
-
kSends
|
499
|
-
};
|
500
|
-
static intptr_t PendingOpMask(PendingOp op) {
|
501
|
-
return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
|
502
|
-
}
|
503
|
-
static std::string PendingOpString(intptr_t pending_ops) {
|
504
|
-
std::vector<absl::string_view> pending_op_strings;
|
505
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
|
506
|
-
pending_op_strings.push_back("kRecvMessage");
|
507
|
-
}
|
508
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
|
509
|
-
pending_op_strings.push_back("kRecvInitialMetadata");
|
510
|
-
}
|
511
|
-
if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
|
512
|
-
pending_op_strings.push_back("kRecvTrailingMetadata");
|
513
|
-
}
|
514
|
-
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
515
|
-
pending_op_strings.push_back("kSends");
|
516
|
-
}
|
517
|
-
return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
|
518
|
-
}
|
519
|
-
struct BatchControl {
|
520
|
-
FilterStackCall* call_ = nullptr;
|
521
|
-
CallTracerAnnotationInterface* call_tracer_ = nullptr;
|
522
|
-
grpc_transport_stream_op_batch op_;
|
523
|
-
// Share memory for cq_completion and notify_tag as they are never needed
|
524
|
-
// simultaneously. Each byte used in this data structure count as six bytes
|
525
|
-
// per call, so any savings we can make are worthwhile,
|
526
|
-
|
527
|
-
// We use notify_tag to determine whether or not to send notification to the
|
528
|
-
// completion queue. Once we've made that determination, we can reuse the
|
529
|
-
// memory for cq_completion.
|
530
|
-
union {
|
531
|
-
grpc_cq_completion cq_completion;
|
532
|
-
struct {
|
533
|
-
// Any given op indicates completion by either (a) calling a closure or
|
534
|
-
// (b) sending a notification on the call's completion queue. If
|
535
|
-
// \a is_closure is true, \a tag indicates a closure to be invoked;
|
536
|
-
// otherwise, \a tag indicates the tag to be used in the notification to
|
537
|
-
// be sent to the completion queue.
|
538
|
-
void* tag;
|
539
|
-
bool is_closure;
|
540
|
-
} notify_tag;
|
541
|
-
} completion_data_;
|
542
|
-
grpc_closure start_batch_;
|
543
|
-
grpc_closure finish_batch_;
|
544
|
-
std::atomic<intptr_t> ops_pending_{0};
|
545
|
-
AtomicError batch_error_;
|
546
|
-
void set_pending_ops(uintptr_t ops) {
|
547
|
-
ops_pending_.store(ops, std::memory_order_release);
|
548
|
-
}
|
549
|
-
bool completed_batch_step(PendingOp op) {
|
550
|
-
auto mask = PendingOpMask(op);
|
551
|
-
auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
|
552
|
-
if (grpc_call_trace.enabled()) {
|
553
|
-
gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
|
554
|
-
PendingOpString(mask).c_str(),
|
555
|
-
PendingOpString(r & ~mask).c_str(),
|
556
|
-
completion_data_.notify_tag.tag);
|
557
|
-
}
|
558
|
-
CHECK_NE((r & mask), 0);
|
559
|
-
return r == mask;
|
560
|
-
}
|
561
|
-
|
562
|
-
void PostCompletion();
|
563
|
-
void FinishStep(PendingOp op);
|
564
|
-
void ProcessDataAfterMetadata();
|
565
|
-
void ReceivingStreamReady(grpc_error_handle error);
|
566
|
-
void ReceivingInitialMetadataReady(grpc_error_handle error);
|
567
|
-
void ReceivingTrailingMetadataReady(grpc_error_handle error);
|
568
|
-
void FinishBatch(grpc_error_handle error);
|
569
|
-
};
|
570
|
-
|
571
|
-
FilterStackCall(Arena* arena, const grpc_call_create_args& args)
|
572
|
-
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
573
|
-
args.channel->Ref()),
|
574
|
-
cq_(args.cq),
|
575
|
-
stream_op_payload_(context_) {
|
576
|
-
context_[GRPC_CONTEXT_CALL].value = this;
|
577
|
-
}
|
578
|
-
|
579
|
-
static void ReleaseCall(void* call, grpc_error_handle);
|
580
|
-
static void DestroyCall(void* call, grpc_error_handle);
|
581
|
-
|
582
|
-
static FilterStackCall* FromCallStack(grpc_call_stack* call_stack) {
|
583
|
-
return reinterpret_cast<FilterStackCall*>(
|
584
|
-
reinterpret_cast<char*>(call_stack) -
|
585
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)));
|
586
|
-
}
|
587
|
-
|
588
|
-
void ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
589
|
-
grpc_closure* start_batch_closure);
|
590
|
-
void SetFinalStatus(grpc_error_handle error);
|
591
|
-
BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
|
592
|
-
bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
|
593
|
-
bool is_trailing);
|
594
|
-
void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
|
595
|
-
void RecvInitialFilter(grpc_metadata_batch* b);
|
596
|
-
void RecvTrailingFilter(grpc_metadata_batch* b,
|
597
|
-
grpc_error_handle batch_error);
|
598
|
-
|
599
|
-
RefCount ext_ref_;
|
600
|
-
CallCombiner call_combiner_;
|
601
|
-
grpc_completion_queue* cq_;
|
602
|
-
grpc_polling_entity pollent_;
|
603
|
-
|
604
|
-
/// has grpc_call_unref been called
|
605
|
-
bool destroy_called_ = false;
|
606
|
-
// Trailers-only response status
|
607
|
-
bool is_trailers_only_ = false;
|
608
|
-
/// which ops are in-flight
|
609
|
-
bool sent_initial_metadata_ = false;
|
610
|
-
bool sending_message_ = false;
|
611
|
-
bool sent_final_op_ = false;
|
612
|
-
bool received_initial_metadata_ = false;
|
613
|
-
bool receiving_message_ = false;
|
614
|
-
bool requested_final_op_ = false;
|
615
|
-
gpr_atm received_final_op_atm_ = 0;
|
616
|
-
|
617
|
-
BatchControl* active_batches_[kMaxConcurrentBatches] = {};
|
618
|
-
grpc_transport_stream_op_batch_payload stream_op_payload_;
|
619
|
-
|
620
|
-
// first idx: is_receiving, second idx: is_trailing
|
621
|
-
grpc_metadata_batch send_initial_metadata_;
|
622
|
-
grpc_metadata_batch send_trailing_metadata_;
|
623
|
-
grpc_metadata_batch recv_initial_metadata_;
|
624
|
-
grpc_metadata_batch recv_trailing_metadata_;
|
625
|
-
|
626
|
-
// Buffered read metadata waiting to be returned to the application.
|
627
|
-
// Element 0 is initial metadata, element 1 is trailing metadata.
|
628
|
-
grpc_metadata_array* buffered_metadata_[2] = {};
|
629
|
-
|
630
|
-
// Call data useful used for reporting. Only valid after the call has
|
631
|
-
// completed
|
632
|
-
grpc_call_final_info final_info_;
|
633
|
-
|
634
|
-
// Contexts for various subsystems (security, tracing, ...).
|
635
|
-
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
636
|
-
|
637
|
-
SliceBuffer send_slice_buffer_;
|
638
|
-
absl::optional<SliceBuffer> receiving_slice_buffer_;
|
639
|
-
uint32_t receiving_stream_flags_;
|
640
|
-
|
641
|
-
bool call_failed_before_recv_message_ = false;
|
642
|
-
grpc_byte_buffer** receiving_buffer_ = nullptr;
|
643
|
-
grpc_slice receiving_slice_ = grpc_empty_slice();
|
644
|
-
grpc_closure receiving_stream_ready_;
|
645
|
-
grpc_closure receiving_initial_metadata_ready_;
|
646
|
-
grpc_closure receiving_trailing_metadata_ready_;
|
647
|
-
// Status about operation of call
|
648
|
-
bool sent_server_trailing_metadata_ = false;
|
649
|
-
gpr_atm cancelled_with_error_ = 0;
|
650
|
-
|
651
|
-
grpc_closure release_call_;
|
652
|
-
|
653
|
-
union {
|
654
|
-
struct {
|
655
|
-
grpc_status_code* status;
|
656
|
-
grpc_slice* status_details;
|
657
|
-
const char** error_string;
|
658
|
-
} client;
|
659
|
-
struct {
|
660
|
-
int* cancelled;
|
661
|
-
// backpointer to owning server if this is a server side call.
|
662
|
-
ServerInterface* core_server;
|
663
|
-
} server;
|
664
|
-
} final_op_;
|
665
|
-
AtomicError status_error_;
|
666
|
-
|
667
|
-
// recv_state can contain one of the following values:
|
668
|
-
// RECV_NONE : : no initial metadata and messages received
|
669
|
-
// RECV_INITIAL_METADATA_FIRST : received initial metadata first
|
670
|
-
// a batch_control* : received messages first
|
671
|
-
|
672
|
-
// +------1------RECV_NONE------3-----+
|
673
|
-
// | |
|
674
|
-
// | |
|
675
|
-
// v v
|
676
|
-
// RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
|
677
|
-
// | ^ | ^
|
678
|
-
// | | | |
|
679
|
-
// +-----2-----+ +-----4-----+
|
680
|
-
|
681
|
-
// For 1, 4: See receiving_initial_metadata_ready() function
|
682
|
-
// For 2, 3: See receiving_stream_ready() function
|
683
|
-
gpr_atm recv_state_ = 0;
|
684
|
-
};
|
685
|
-
|
686
|
-
grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
687
|
-
grpc_call** out_call) {
|
688
|
-
Channel* channel = args->channel.get();
|
689
|
-
|
690
|
-
auto add_init_error = [](grpc_error_handle* composite,
|
691
|
-
grpc_error_handle new_err) {
|
692
|
-
if (new_err.ok()) return;
|
693
|
-
if (composite->ok()) {
|
694
|
-
*composite = GRPC_ERROR_CREATE("Call creation failed");
|
695
|
-
}
|
696
|
-
*composite = grpc_error_add_child(*composite, new_err);
|
697
|
-
};
|
698
|
-
|
699
|
-
FilterStackCall* call;
|
700
|
-
grpc_error_handle error;
|
701
|
-
grpc_channel_stack* channel_stack = channel->channel_stack();
|
702
|
-
size_t call_alloc_size =
|
703
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
|
704
|
-
channel_stack->call_stack_size;
|
705
|
-
|
706
|
-
Arena* arena = channel->CreateArena();
|
707
|
-
call = new (arena->Alloc(call_alloc_size)) FilterStackCall(arena, *args);
|
708
|
-
DCHECK(FromC(call->c_ptr()) == call);
|
709
|
-
DCHECK(FromCallStack(call->call_stack()) == call);
|
710
|
-
*out_call = call->c_ptr();
|
711
|
-
grpc_slice path = grpc_empty_slice();
|
712
|
-
ScopedContext ctx(call);
|
713
|
-
if (call->is_client()) {
|
714
|
-
call->final_op_.client.status_details = nullptr;
|
715
|
-
call->final_op_.client.status = nullptr;
|
716
|
-
call->final_op_.client.error_string = nullptr;
|
717
|
-
global_stats().IncrementClientCallsCreated();
|
718
|
-
path = CSliceRef(args->path->c_slice());
|
719
|
-
call->send_initial_metadata_.Set(HttpPathMetadata(),
|
720
|
-
std::move(*args->path));
|
721
|
-
if (args->authority.has_value()) {
|
722
|
-
call->send_initial_metadata_.Set(HttpAuthorityMetadata(),
|
723
|
-
std::move(*args->authority));
|
724
|
-
}
|
725
|
-
call->send_initial_metadata_.Set(
|
726
|
-
GrpcRegisteredMethod(), reinterpret_cast<void*>(static_cast<uintptr_t>(
|
727
|
-
args->registered_method)));
|
728
|
-
channel_stack->stats_plugin_group->AddClientCallTracers(
|
729
|
-
Slice(CSliceRef(path)), args->registered_method, call->context_);
|
730
|
-
} else {
|
731
|
-
global_stats().IncrementServerCallsCreated();
|
732
|
-
call->final_op_.server.cancelled = nullptr;
|
733
|
-
call->final_op_.server.core_server = args->server;
|
734
|
-
// TODO(yashykt): In the future, we want to also enable stats and trace
|
735
|
-
// collecting from when the call is created at the transport. The idea is
|
736
|
-
// that the transport would create the call tracer and pass it in as part of
|
737
|
-
// the metadata.
|
738
|
-
// TODO(yijiem): OpenCensus and internal Census is still using this way to
|
739
|
-
// set server call tracer. We need to refactor them to stats plugins
|
740
|
-
// (including removing the client channel filters).
|
741
|
-
if (args->server != nullptr &&
|
742
|
-
args->server->server_call_tracer_factory() != nullptr) {
|
743
|
-
auto* server_call_tracer =
|
744
|
-
args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
|
745
|
-
arena, args->server->channel_args());
|
746
|
-
if (server_call_tracer != nullptr) {
|
747
|
-
// Note that we are setting both
|
748
|
-
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
749
|
-
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
750
|
-
// promise-based world, we would just a single tracer object for each
|
751
|
-
// stack (call, subchannel_call, server_call.)
|
752
|
-
call->ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
753
|
-
server_call_tracer, nullptr);
|
754
|
-
call->ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
755
|
-
}
|
756
|
-
}
|
757
|
-
channel_stack->stats_plugin_group->AddServerCallTracers(call->context_);
|
758
|
-
}
|
759
|
-
|
760
|
-
Call* parent = Call::FromC(args->parent);
|
761
|
-
if (parent != nullptr) {
|
762
|
-
add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
|
763
|
-
parent, args->propagation_mask)));
|
764
|
-
}
|
765
|
-
// initial refcount dropped by grpc_call_unref
|
766
|
-
grpc_call_element_args call_args = {
|
767
|
-
call->call_stack(), args->server_transport_data,
|
768
|
-
call->context_, path,
|
769
|
-
call->start_time(), call->send_deadline(),
|
770
|
-
call->arena(), &call->call_combiner_};
|
771
|
-
add_init_error(&error, grpc_call_stack_init(channel_stack, 1, DestroyCall,
|
772
|
-
call, &call_args));
|
773
|
-
// Publish this call to parent only after the call stack has been initialized.
|
774
|
-
if (parent != nullptr) {
|
775
|
-
call->PublishToParent(parent);
|
776
|
-
}
|
777
|
-
|
778
|
-
if (!error.ok()) {
|
779
|
-
call->CancelWithError(error);
|
780
|
-
}
|
781
|
-
if (args->cq != nullptr) {
|
782
|
-
CHECK(args->pollset_set_alternative == nullptr)
|
783
|
-
<< "Only one of 'cq' and 'pollset_set_alternative' should be "
|
784
|
-
"non-nullptr.";
|
785
|
-
GRPC_CQ_INTERNAL_REF(args->cq, "bind");
|
786
|
-
call->pollent_ =
|
787
|
-
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args->cq));
|
788
|
-
}
|
789
|
-
if (args->pollset_set_alternative != nullptr) {
|
790
|
-
call->pollent_ = grpc_polling_entity_create_from_pollset_set(
|
791
|
-
args->pollset_set_alternative);
|
792
|
-
}
|
793
|
-
if (!grpc_polling_entity_is_empty(&call->pollent_)) {
|
794
|
-
grpc_call_stack_set_pollset_or_pollset_set(call->call_stack(),
|
795
|
-
&call->pollent_);
|
796
|
-
}
|
797
|
-
|
798
|
-
if (call->is_client()) {
|
799
|
-
channelz::ChannelNode* channelz_channel = channel->channelz_node();
|
800
|
-
if (channelz_channel != nullptr) {
|
801
|
-
channelz_channel->RecordCallStarted();
|
802
|
-
}
|
803
|
-
} else if (call->final_op_.server.core_server != nullptr) {
|
804
|
-
channelz::ServerNode* channelz_node =
|
805
|
-
call->final_op_.server.core_server->channelz_node();
|
806
|
-
if (channelz_node != nullptr) {
|
807
|
-
channelz_node->RecordCallStarted();
|
808
|
-
}
|
809
|
-
}
|
810
|
-
|
811
|
-
if (args->send_deadline != Timestamp::InfFuture()) {
|
812
|
-
call->UpdateDeadline(args->send_deadline);
|
813
|
-
}
|
814
|
-
|
815
|
-
CSliceUnref(path);
|
816
|
-
|
817
|
-
return error;
|
818
|
-
}
|
819
|
-
|
820
|
-
void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
821
|
-
CHECK(cq);
|
822
|
-
|
823
|
-
if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
|
824
|
-
Crash("A pollset_set is already registered for this call.");
|
825
|
-
}
|
826
|
-
cq_ = cq;
|
827
|
-
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
828
|
-
pollent_ = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
|
829
|
-
grpc_call_stack_set_pollset_or_pollset_set(call_stack(), &pollent_);
|
830
|
-
}
|
831
|
-
|
832
|
-
void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
|
833
|
-
static_cast<FilterStackCall*>(call)->DeleteThis();
|
834
|
-
}
|
835
|
-
|
836
|
-
void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
837
|
-
auto* c = static_cast<FilterStackCall*>(call);
|
838
|
-
c->recv_initial_metadata_.Clear();
|
839
|
-
c->recv_trailing_metadata_.Clear();
|
840
|
-
c->receiving_slice_buffer_.reset();
|
841
|
-
ParentCall* pc = c->parent_call();
|
842
|
-
if (pc != nullptr) {
|
843
|
-
pc->~ParentCall();
|
844
|
-
}
|
845
|
-
if (c->cq_) {
|
846
|
-
GRPC_CQ_INTERNAL_UNREF(c->cq_, "bind");
|
847
|
-
}
|
848
|
-
|
849
|
-
grpc_error_handle status_error = c->status_error_.get();
|
850
|
-
grpc_error_get_status(status_error, c->send_deadline(),
|
851
|
-
&c->final_info_.final_status, nullptr, nullptr,
|
852
|
-
&(c->final_info_.error_string));
|
853
|
-
c->status_error_.set(absl::OkStatus());
|
854
|
-
c->final_info_.stats.latency =
|
855
|
-
gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time());
|
856
|
-
grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
|
857
|
-
GRPC_CLOSURE_INIT(&c->release_call_, ReleaseCall, c,
|
858
|
-
grpc_schedule_on_exec_ctx));
|
859
|
-
}
|
860
|
-
|
861
|
-
void FilterStackCall::ExternalUnref() {
|
862
|
-
if (GPR_LIKELY(!ext_ref_.Unref())) return;
|
863
|
-
|
864
|
-
ApplicationCallbackExecCtx callback_exec_ctx;
|
865
|
-
ExecCtx exec_ctx;
|
866
|
-
|
867
|
-
GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (this));
|
868
|
-
|
869
|
-
MaybeUnpublishFromParent();
|
870
|
-
|
871
|
-
CHECK(!destroy_called_);
|
872
|
-
destroy_called_ = true;
|
873
|
-
bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
|
874
|
-
if (cancel) {
|
875
|
-
CancelWithError(absl::CancelledError());
|
876
|
-
} else {
|
877
|
-
// Unset the call combiner cancellation closure. This has the
|
878
|
-
// effect of scheduling the previously set cancellation closure, if
|
879
|
-
// any, so that it can release any internal references it may be
|
880
|
-
// holding to the call stack.
|
881
|
-
call_combiner_.SetNotifyOnCancel(nullptr);
|
882
|
-
}
|
883
|
-
InternalUnref("destroy");
|
884
|
-
}
|
885
|
-
|
886
|
-
// start_batch_closure points to a caller-allocated closure to be used
|
887
|
-
// for entering the call combiner.
|
888
|
-
void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
889
|
-
grpc_closure* start_batch_closure) {
|
890
|
-
// This is called via the call combiner to start sending a batch down
|
891
|
-
// the filter stack.
|
892
|
-
auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
|
893
|
-
grpc_transport_stream_op_batch* batch =
|
894
|
-
static_cast<grpc_transport_stream_op_batch*>(arg);
|
895
|
-
auto* call =
|
896
|
-
static_cast<FilterStackCall*>(batch->handler_private.extra_arg);
|
897
|
-
grpc_call_element* elem = call->call_elem(0);
|
898
|
-
GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
|
899
|
-
elem->filter->start_transport_stream_op_batch(elem, batch);
|
900
|
-
};
|
901
|
-
batch->handler_private.extra_arg = this;
|
902
|
-
GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
|
903
|
-
grpc_schedule_on_exec_ctx);
|
904
|
-
GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
|
905
|
-
absl::OkStatus(), "executing batch");
|
906
|
-
}
|
907
|
-
|
908
|
-
namespace {
|
909
|
-
struct CancelState {
|
910
|
-
FilterStackCall* call;
|
911
|
-
grpc_closure start_batch;
|
912
|
-
grpc_closure finish_batch;
|
913
|
-
};
|
914
|
-
} // namespace
|
915
|
-
|
916
|
-
// The on_complete callback used when sending a cancel_stream batch down
|
917
|
-
// the filter stack. Yields the call combiner when the batch is done.
|
918
|
-
static void done_termination(void* arg, grpc_error_handle /*error*/) {
|
919
|
-
CancelState* state = static_cast<CancelState*>(arg);
|
920
|
-
GRPC_CALL_COMBINER_STOP(state->call->call_combiner(),
|
921
|
-
"on_complete for cancel_stream op");
|
922
|
-
state->call->InternalUnref("termination");
|
923
|
-
delete state;
|
924
|
-
}
|
925
|
-
|
926
|
-
void FilterStackCall::CancelWithError(grpc_error_handle error) {
|
927
|
-
if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
|
928
|
-
return;
|
929
|
-
}
|
930
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
931
|
-
gpr_log(GPR_INFO, "CancelWithError %s %s", is_client() ? "CLI" : "SVR",
|
932
|
-
StatusToString(error).c_str());
|
933
|
-
}
|
934
|
-
ClearPeerString();
|
935
|
-
InternalRef("termination");
|
936
|
-
ResetDeadline();
|
937
|
-
// Inform the call combiner of the cancellation, so that it can cancel
|
938
|
-
// any in-flight asynchronous actions that may be holding the call
|
939
|
-
// combiner. This ensures that the cancel_stream batch can be sent
|
940
|
-
// down the filter stack in a timely manner.
|
941
|
-
call_combiner_.Cancel(error);
|
942
|
-
CancelState* state = new CancelState;
|
943
|
-
state->call = this;
|
944
|
-
GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
|
945
|
-
grpc_schedule_on_exec_ctx);
|
946
|
-
grpc_transport_stream_op_batch* op =
|
947
|
-
grpc_make_transport_stream_op(&state->finish_batch);
|
948
|
-
op->cancel_stream = true;
|
949
|
-
op->payload->cancel_stream.cancel_error = error;
|
950
|
-
ExecuteBatch(op, &state->start_batch);
|
951
|
-
}
|
952
|
-
|
953
|
-
void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
954
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
955
|
-
gpr_log(GPR_INFO, "set_final_status %s %s", is_client() ? "CLI" : "SVR",
|
956
|
-
StatusToString(error).c_str());
|
957
|
-
}
|
958
|
-
ResetDeadline();
|
959
|
-
if (is_client()) {
|
960
|
-
std::string status_details;
|
961
|
-
grpc_error_get_status(error, send_deadline(), final_op_.client.status,
|
962
|
-
&status_details, nullptr,
|
963
|
-
final_op_.client.error_string);
|
964
|
-
*final_op_.client.status_details =
|
965
|
-
grpc_slice_from_cpp_string(std::move(status_details));
|
966
|
-
status_error_.set(error);
|
967
|
-
channelz::ChannelNode* channelz_channel = channel()->channelz_node();
|
968
|
-
if (channelz_channel != nullptr) {
|
969
|
-
if (*final_op_.client.status != GRPC_STATUS_OK) {
|
970
|
-
channelz_channel->RecordCallFailed();
|
971
|
-
} else {
|
972
|
-
channelz_channel->RecordCallSucceeded();
|
973
|
-
}
|
974
|
-
}
|
975
|
-
} else {
|
976
|
-
*final_op_.server.cancelled =
|
977
|
-
!error.ok() || !sent_server_trailing_metadata_;
|
978
|
-
channelz::ServerNode* channelz_node =
|
979
|
-
final_op_.server.core_server->channelz_node();
|
980
|
-
if (channelz_node != nullptr) {
|
981
|
-
if (*final_op_.server.cancelled || !status_error_.ok()) {
|
982
|
-
channelz_node->RecordCallFailed();
|
983
|
-
} else {
|
984
|
-
channelz_node->RecordCallSucceeded();
|
985
|
-
}
|
986
|
-
}
|
987
|
-
}
|
988
|
-
}
|
989
|
-
|
990
|
-
bool FilterStackCall::PrepareApplicationMetadata(size_t count,
|
991
|
-
grpc_metadata* metadata,
|
992
|
-
bool is_trailing) {
|
993
|
-
grpc_metadata_batch* batch =
|
994
|
-
is_trailing ? &send_trailing_metadata_ : &send_initial_metadata_;
|
995
|
-
for (size_t i = 0; i < count; i++) {
|
996
|
-
grpc_metadata* md = &metadata[i];
|
997
|
-
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
998
|
-
grpc_validate_header_key_is_legal(md->key))) {
|
999
|
-
return false;
|
1000
|
-
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1001
|
-
!GRPC_LOG_IF_ERROR(
|
1002
|
-
"validate_metadata",
|
1003
|
-
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1004
|
-
return false;
|
1005
|
-
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1006
|
-
// HTTP2 hpack encoding has a maximum limit.
|
1007
|
-
return false;
|
1008
|
-
} else if (grpc_slice_str_cmp(md->key, "content-length") == 0) {
|
1009
|
-
// Filter "content-length metadata"
|
1010
|
-
continue;
|
1011
|
-
}
|
1012
|
-
batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
|
1013
|
-
[md](absl::string_view error, const Slice& value) {
|
1014
|
-
gpr_log(GPR_DEBUG, "Append error: %s",
|
1015
|
-
absl::StrCat("key=", StringViewFromSlice(md->key),
|
1016
|
-
" error=", error,
|
1017
|
-
" value=", value.as_string_view())
|
1018
|
-
.c_str());
|
1019
|
-
});
|
1020
|
-
}
|
1021
|
-
|
1022
|
-
return true;
|
1023
|
-
}
|
1024
|
-
|
1025
|
-
namespace {
|
1026
|
-
class PublishToAppEncoder {
|
1027
|
-
public:
|
1028
|
-
explicit PublishToAppEncoder(grpc_metadata_array* dest,
|
1029
|
-
const grpc_metadata_batch* encoding,
|
1030
|
-
bool is_client)
|
1031
|
-
: dest_(dest), encoding_(encoding), is_client_(is_client) {}
|
1032
|
-
|
1033
|
-
void Encode(const Slice& key, const Slice& value) {
|
1034
|
-
Append(key.c_slice(), value.c_slice());
|
1035
|
-
}
|
1036
|
-
|
1037
|
-
// Catch anything that is not explicitly handled, and do not publish it to the
|
1038
|
-
// application. If new metadata is added to a batch that needs to be
|
1039
|
-
// published, it should be called out here.
|
1040
|
-
template <typename Which>
|
1041
|
-
void Encode(Which, const typename Which::ValueType&) {}
|
1042
|
-
|
1043
|
-
void Encode(UserAgentMetadata, const Slice& slice) {
|
1044
|
-
Append(UserAgentMetadata::key(), slice);
|
1045
|
-
}
|
1046
|
-
|
1047
|
-
void Encode(HostMetadata, const Slice& slice) {
|
1048
|
-
Append(HostMetadata::key(), slice);
|
1049
|
-
}
|
1050
|
-
|
1051
|
-
void Encode(GrpcPreviousRpcAttemptsMetadata, uint32_t count) {
|
1052
|
-
Append(GrpcPreviousRpcAttemptsMetadata::key(), count);
|
1053
|
-
}
|
1054
|
-
|
1055
|
-
void Encode(GrpcRetryPushbackMsMetadata, Duration count) {
|
1056
|
-
Append(GrpcRetryPushbackMsMetadata::key(), count.millis());
|
1057
|
-
}
|
1058
|
-
|
1059
|
-
void Encode(LbTokenMetadata, const Slice& slice) {
|
1060
|
-
Append(LbTokenMetadata::key(), slice);
|
1061
|
-
}
|
1062
|
-
|
1063
|
-
private:
|
1064
|
-
void Append(absl::string_view key, int64_t value) {
|
1065
|
-
Append(StaticSlice::FromStaticString(key).c_slice(),
|
1066
|
-
Slice::FromInt64(value).c_slice());
|
1067
|
-
}
|
1068
|
-
|
1069
|
-
void Append(absl::string_view key, const Slice& value) {
|
1070
|
-
Append(StaticSlice::FromStaticString(key).c_slice(), value.c_slice());
|
1071
|
-
}
|
1072
|
-
|
1073
|
-
void Append(grpc_slice key, grpc_slice value) {
|
1074
|
-
if (dest_->count == dest_->capacity) {
|
1075
|
-
Crash(absl::StrCat(
|
1076
|
-
"Too many metadata entries: capacity=", dest_->capacity, " on ",
|
1077
|
-
is_client_ ? "client" : "server", " encoding ", encoding_->count(),
|
1078
|
-
" elements: ", encoding_->DebugString().c_str()));
|
1079
|
-
}
|
1080
|
-
auto* mdusr = &dest_->metadata[dest_->count++];
|
1081
|
-
mdusr->key = key;
|
1082
|
-
mdusr->value = value;
|
1083
|
-
}
|
1084
|
-
|
1085
|
-
grpc_metadata_array* const dest_;
|
1086
|
-
const grpc_metadata_batch* const encoding_;
|
1087
|
-
const bool is_client_;
|
1088
|
-
};
|
1089
|
-
} // namespace
|
1090
|
-
|
1091
|
-
void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
|
1092
|
-
bool is_trailing) {
|
1093
|
-
if (b->count() == 0) return;
|
1094
|
-
if (!is_client() && is_trailing) return;
|
1095
|
-
if (is_trailing && buffered_metadata_[1] == nullptr) return;
|
1096
|
-
grpc_metadata_array* dest;
|
1097
|
-
dest = buffered_metadata_[is_trailing];
|
1098
|
-
if (dest->count + b->count() > dest->capacity) {
|
1099
|
-
dest->capacity =
|
1100
|
-
std::max(dest->capacity + b->count(), dest->capacity * 3 / 2);
|
1101
|
-
dest->metadata = static_cast<grpc_metadata*>(
|
1102
|
-
gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity));
|
1103
|
-
}
|
1104
|
-
PublishToAppEncoder encoder(dest, b, is_client());
|
1105
|
-
b->Encode(&encoder);
|
1106
|
-
}
|
1107
|
-
|
1108
|
-
void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
|
1109
|
-
ProcessIncomingInitialMetadata(*b);
|
1110
|
-
PublishAppMetadata(b, false);
|
1111
|
-
}
|
1112
|
-
|
1113
|
-
void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
1114
|
-
grpc_error_handle batch_error) {
|
1115
|
-
if (!batch_error.ok()) {
|
1116
|
-
SetFinalStatus(batch_error);
|
1117
|
-
} else {
|
1118
|
-
absl::optional<grpc_status_code> grpc_status =
|
1119
|
-
b->Take(GrpcStatusMetadata());
|
1120
|
-
if (grpc_status.has_value()) {
|
1121
|
-
grpc_status_code status_code = *grpc_status;
|
1122
|
-
grpc_error_handle error;
|
1123
|
-
if (status_code != GRPC_STATUS_OK) {
|
1124
|
-
Slice peer = GetPeerString();
|
1125
|
-
error = grpc_error_set_int(
|
1126
|
-
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
|
1127
|
-
peer.as_string_view())),
|
1128
|
-
StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
|
1129
|
-
}
|
1130
|
-
auto grpc_message = b->Take(GrpcMessageMetadata());
|
1131
|
-
if (grpc_message.has_value()) {
|
1132
|
-
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
|
1133
|
-
grpc_message->as_string_view());
|
1134
|
-
} else if (!error.ok()) {
|
1135
|
-
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
|
1136
|
-
}
|
1137
|
-
SetFinalStatus(error);
|
1138
|
-
} else if (!is_client()) {
|
1139
|
-
SetFinalStatus(absl::OkStatus());
|
1140
|
-
} else {
|
1141
|
-
gpr_log(GPR_DEBUG,
|
1142
|
-
"Received trailing metadata with no error and no status");
|
1143
|
-
SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
|
1144
|
-
StatusIntProperty::kRpcStatus,
|
1145
|
-
GRPC_STATUS_UNKNOWN));
|
1146
|
-
}
|
1147
|
-
}
|
1148
|
-
PublishAppMetadata(b, true);
|
1149
|
-
}
|
1150
|
-
|
1151
|
-
namespace {
|
1152
|
-
bool AreWriteFlagsValid(uint32_t flags) {
|
1153
|
-
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
1154
|
-
const uint32_t allowed_write_positions =
|
1155
|
-
(GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
|
1156
|
-
const uint32_t invalid_positions = ~allowed_write_positions;
|
1157
|
-
return !(flags & invalid_positions);
|
1158
|
-
}
|
1159
|
-
|
1160
|
-
bool AreInitialMetadataFlagsValid(uint32_t flags) {
|
1161
|
-
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
1162
|
-
uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
|
1163
|
-
return !(flags & invalid_positions);
|
1164
|
-
}
|
1165
|
-
|
1166
|
-
size_t BatchSlotForOp(grpc_op_type type) {
|
1167
|
-
switch (type) {
|
1168
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
1169
|
-
return 0;
|
1170
|
-
case GRPC_OP_SEND_MESSAGE:
|
1171
|
-
return 1;
|
1172
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
1173
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
1174
|
-
return 2;
|
1175
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
1176
|
-
return 3;
|
1177
|
-
case GRPC_OP_RECV_MESSAGE:
|
1178
|
-
return 4;
|
1179
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
1180
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
1181
|
-
return 5;
|
1182
|
-
}
|
1183
|
-
GPR_UNREACHABLE_CODE(return 123456789);
|
1184
|
-
}
|
1185
|
-
} // namespace
|
1186
|
-
|
1187
|
-
FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
|
1188
|
-
const grpc_op* ops) {
|
1189
|
-
size_t slot_idx = BatchSlotForOp(ops[0].op);
|
1190
|
-
BatchControl** pslot = &active_batches_[slot_idx];
|
1191
|
-
BatchControl* bctl;
|
1192
|
-
if (*pslot != nullptr) {
|
1193
|
-
bctl = *pslot;
|
1194
|
-
if (bctl->call_ != nullptr) {
|
1195
|
-
return nullptr;
|
1196
|
-
}
|
1197
|
-
bctl->~BatchControl();
|
1198
|
-
bctl->op_ = {};
|
1199
|
-
new (&bctl->batch_error_) AtomicError();
|
1200
|
-
} else {
|
1201
|
-
bctl = arena()->New<BatchControl>();
|
1202
|
-
*pslot = bctl;
|
1203
|
-
}
|
1204
|
-
bctl->call_ = this;
|
1205
|
-
bctl->call_tracer_ = static_cast<CallTracerAnnotationInterface*>(
|
1206
|
-
ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
|
1207
|
-
bctl->op_.payload = &stream_op_payload_;
|
1208
|
-
return bctl;
|
1209
|
-
}
|
1210
|
-
|
1211
|
-
void FilterStackCall::BatchControl::PostCompletion() {
|
1212
|
-
FilterStackCall* call = call_;
|
1213
|
-
grpc_error_handle error = batch_error_.get();
|
1214
|
-
|
1215
|
-
if (IsCallStatusOverrideOnCancellationEnabled()) {
|
1216
|
-
// On the client side, if final call status is already known (i.e if this op
|
1217
|
-
// includes recv_trailing_metadata) and if the call status is known to be
|
1218
|
-
// OK, then disregard the batch error to ensure call->receiving_buffer_ is
|
1219
|
-
// not cleared.
|
1220
|
-
if (op_.recv_trailing_metadata && call->is_client() &&
|
1221
|
-
call->status_error_.ok()) {
|
1222
|
-
error = absl::OkStatus();
|
1223
|
-
}
|
1224
|
-
}
|
1225
|
-
|
1226
|
-
if (grpc_call_trace.enabled()) {
|
1227
|
-
gpr_log(GPR_DEBUG, "tag:%p batch_error=%s op:%s",
|
1228
|
-
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1229
|
-
grpc_transport_stream_op_batch_string(&op_, false).c_str());
|
1230
|
-
}
|
1231
|
-
|
1232
|
-
if (op_.send_initial_metadata) {
|
1233
|
-
call->send_initial_metadata_.Clear();
|
1234
|
-
}
|
1235
|
-
if (op_.send_message) {
|
1236
|
-
if (op_.payload->send_message.stream_write_closed && error.ok()) {
|
1237
|
-
error = grpc_error_add_child(
|
1238
|
-
error, GRPC_ERROR_CREATE(
|
1239
|
-
"Attempt to send message after stream was closed."));
|
1240
|
-
}
|
1241
|
-
call->sending_message_ = false;
|
1242
|
-
call->send_slice_buffer_.Clear();
|
1243
|
-
}
|
1244
|
-
if (op_.send_trailing_metadata) {
|
1245
|
-
call->send_trailing_metadata_.Clear();
|
1246
|
-
}
|
1247
|
-
|
1248
|
-
if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
|
1249
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1250
|
-
*call->receiving_buffer_ = nullptr;
|
1251
|
-
}
|
1252
|
-
if (op_.recv_trailing_metadata) {
|
1253
|
-
// propagate cancellation to any interested children
|
1254
|
-
gpr_atm_rel_store(&call->received_final_op_atm_, 1);
|
1255
|
-
call->PropagateCancellationToChildren();
|
1256
|
-
error = absl::OkStatus();
|
1257
|
-
}
|
1258
|
-
batch_error_.set(absl::OkStatus());
|
1259
|
-
|
1260
|
-
if (completion_data_.notify_tag.is_closure) {
|
1261
|
-
call_ = nullptr;
|
1262
|
-
GrpcClosure::Run(
|
1263
|
-
DEBUG_LOCATION,
|
1264
|
-
static_cast<grpc_closure*>(completion_data_.notify_tag.tag), error);
|
1265
|
-
call->InternalUnref("completion");
|
1266
|
-
} else {
|
1267
|
-
grpc_cq_end_op(
|
1268
|
-
call->cq_, completion_data_.notify_tag.tag, error,
|
1269
|
-
[](void* user_data, grpc_cq_completion* /*storage*/) {
|
1270
|
-
BatchControl* bctl = static_cast<BatchControl*>(user_data);
|
1271
|
-
Call* call = bctl->call_;
|
1272
|
-
bctl->call_ = nullptr;
|
1273
|
-
call->InternalUnref("completion");
|
1274
|
-
},
|
1275
|
-
this, &completion_data_.cq_completion);
|
1276
|
-
}
|
1277
|
-
}
|
1278
|
-
|
1279
|
-
void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
|
1280
|
-
if (GPR_UNLIKELY(completed_batch_step(op))) {
|
1281
|
-
PostCompletion();
|
1282
|
-
}
|
1283
|
-
}
|
1284
|
-
|
1285
|
-
void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
|
1286
|
-
FilterStackCall* call = call_;
|
1287
|
-
if (!call->receiving_slice_buffer_.has_value()) {
|
1288
|
-
*call->receiving_buffer_ = nullptr;
|
1289
|
-
call->receiving_message_ = false;
|
1290
|
-
FinishStep(PendingOp::kRecvMessage);
|
1291
|
-
} else {
|
1292
|
-
call->NoteLastMessageFlags(call->receiving_stream_flags_);
|
1293
|
-
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
1294
|
-
(call->incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
1295
|
-
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
|
1296
|
-
nullptr, 0, call->incoming_compression_algorithm());
|
1297
|
-
} else {
|
1298
|
-
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
1299
|
-
}
|
1300
|
-
grpc_slice_buffer_move_into(
|
1301
|
-
call->receiving_slice_buffer_->c_slice_buffer(),
|
1302
|
-
&(*call->receiving_buffer_)->data.raw.slice_buffer);
|
1303
|
-
call->receiving_message_ = false;
|
1304
|
-
call->receiving_slice_buffer_.reset();
|
1305
|
-
FinishStep(PendingOp::kRecvMessage);
|
1306
|
-
}
|
1307
|
-
}
|
1308
|
-
|
1309
|
-
void FilterStackCall::BatchControl::ReceivingStreamReady(
|
1310
|
-
grpc_error_handle error) {
|
1311
|
-
if (grpc_call_trace.enabled()) {
|
1312
|
-
gpr_log(GPR_DEBUG,
|
1313
|
-
"tag:%p ReceivingStreamReady error=%s "
|
1314
|
-
"receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
|
1315
|
-
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1316
|
-
call_->receiving_slice_buffer_.has_value(),
|
1317
|
-
gpr_atm_no_barrier_load(&call_->recv_state_));
|
1318
|
-
}
|
1319
|
-
FilterStackCall* call = call_;
|
1320
|
-
if (!error.ok()) {
|
1321
|
-
call->receiving_slice_buffer_.reset();
|
1322
|
-
if (batch_error_.ok()) {
|
1323
|
-
batch_error_.set(error);
|
1324
|
-
}
|
1325
|
-
call->CancelWithError(error);
|
1326
|
-
}
|
1327
|
-
// If recv_state is kRecvNone, we will save the batch_control
|
1328
|
-
// object with rel_cas, and will not use it after the cas. Its corresponding
|
1329
|
-
// acq_load is in receiving_initial_metadata_ready()
|
1330
|
-
if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
|
1331
|
-
!gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
|
1332
|
-
reinterpret_cast<gpr_atm>(this))) {
|
1333
|
-
ProcessDataAfterMetadata();
|
1334
|
-
}
|
1335
|
-
}
|
1336
|
-
|
1337
|
-
void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
1338
|
-
grpc_error_handle error) {
|
1339
|
-
FilterStackCall* call = call_;
|
1340
|
-
|
1341
|
-
GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
|
1342
|
-
|
1343
|
-
if (error.ok()) {
|
1344
|
-
grpc_metadata_batch* md = &call->recv_initial_metadata_;
|
1345
|
-
call->RecvInitialFilter(md);
|
1346
|
-
|
1347
|
-
absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
|
1348
|
-
if (deadline.has_value() && !call->is_client()) {
|
1349
|
-
call_->set_send_deadline(*deadline);
|
1350
|
-
}
|
1351
|
-
} else {
|
1352
|
-
if (batch_error_.ok()) {
|
1353
|
-
batch_error_.set(error);
|
1354
|
-
}
|
1355
|
-
call->CancelWithError(error);
|
1356
|
-
}
|
384
|
+
} // namespace grpc_core
|
1357
385
|
|
1358
|
-
|
1359
|
-
|
1360
|
-
gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
|
1361
|
-
// Should only receive initial metadata once
|
1362
|
-
CHECK_NE(rsr_bctlp, 1);
|
1363
|
-
if (rsr_bctlp == 0) {
|
1364
|
-
// We haven't seen initial metadata and messages before, thus initial
|
1365
|
-
// metadata is received first.
|
1366
|
-
// no_barrier_cas is used, as this function won't access the batch_control
|
1367
|
-
// object saved by receiving_stream_ready() if the initial metadata is
|
1368
|
-
// received first.
|
1369
|
-
if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
|
1370
|
-
kRecvInitialMetadataFirst)) {
|
1371
|
-
break;
|
1372
|
-
}
|
1373
|
-
} else {
|
1374
|
-
// Already received messages
|
1375
|
-
saved_rsr_closure = GRPC_CLOSURE_CREATE(
|
1376
|
-
[](void* bctl, grpc_error_handle error) {
|
1377
|
-
static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
|
1378
|
-
},
|
1379
|
-
reinterpret_cast<BatchControl*>(rsr_bctlp),
|
1380
|
-
grpc_schedule_on_exec_ctx);
|
1381
|
-
// No need to modify recv_state
|
1382
|
-
break;
|
1383
|
-
}
|
1384
|
-
}
|
1385
|
-
if (saved_rsr_closure != nullptr) {
|
1386
|
-
GrpcClosure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
|
1387
|
-
}
|
386
|
+
///////////////////////////////////////////////////////////////////////////////
|
387
|
+
// C-based API
|
1388
388
|
|
1389
|
-
|
389
|
+
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
390
|
+
grpc_core::ExecCtx exec_ctx;
|
391
|
+
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
1390
392
|
}
|
1391
393
|
|
1392
|
-
void
|
1393
|
-
|
1394
|
-
|
1395
|
-
"recv_trailing_metadata_ready");
|
1396
|
-
grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
|
1397
|
-
call_->RecvTrailingFilter(md, error);
|
1398
|
-
FinishStep(PendingOp::kRecvTrailingMetadata);
|
394
|
+
void grpc_call_set_completion_queue(grpc_call* call,
|
395
|
+
grpc_completion_queue* cq) {
|
396
|
+
grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
|
1399
397
|
}
|
1400
398
|
|
1401
|
-
void
|
1402
|
-
GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "on_complete");
|
1403
|
-
if (batch_error_.ok()) {
|
1404
|
-
batch_error_.set(error);
|
1405
|
-
}
|
1406
|
-
if (!error.ok()) {
|
1407
|
-
call_->CancelWithError(error);
|
1408
|
-
}
|
1409
|
-
FinishStep(PendingOp::kSends);
|
1410
|
-
}
|
399
|
+
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
1411
400
|
|
1412
|
-
|
1413
|
-
|
1414
|
-
|
1415
|
-
if (!is_notify_tag_closure) {
|
1416
|
-
CHECK(grpc_cq_begin_op(cq, notify_tag));
|
1417
|
-
grpc_cq_end_op(
|
1418
|
-
cq, notify_tag, absl::OkStatus(),
|
1419
|
-
[](void*, grpc_cq_completion* completion) { gpr_free(completion); },
|
1420
|
-
nullptr,
|
1421
|
-
static_cast<grpc_cq_completion*>(
|
1422
|
-
gpr_malloc(sizeof(grpc_cq_completion))));
|
1423
|
-
} else {
|
1424
|
-
Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
|
1425
|
-
absl::OkStatus());
|
1426
|
-
}
|
1427
|
-
}
|
1428
|
-
} // namespace
|
1429
|
-
|
1430
|
-
grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
1431
|
-
void* notify_tag,
|
1432
|
-
bool is_notify_tag_closure) {
|
1433
|
-
size_t i;
|
1434
|
-
const grpc_op* op;
|
1435
|
-
BatchControl* bctl;
|
1436
|
-
grpc_call_error error = GRPC_CALL_OK;
|
1437
|
-
grpc_transport_stream_op_batch* stream_op;
|
1438
|
-
grpc_transport_stream_op_batch_payload* stream_op_payload;
|
1439
|
-
uint32_t seen_ops = 0;
|
1440
|
-
intptr_t pending_ops = 0;
|
1441
|
-
|
1442
|
-
for (i = 0; i < nops; i++) {
|
1443
|
-
if (seen_ops & (1u << ops[i].op)) {
|
1444
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1445
|
-
}
|
1446
|
-
seen_ops |= (1u << ops[i].op);
|
1447
|
-
}
|
1448
|
-
|
1449
|
-
if (!is_client() &&
|
1450
|
-
(seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
|
1451
|
-
(seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
|
1452
|
-
gpr_log(GPR_ERROR,
|
1453
|
-
"******************* SEND_STATUS WITH RECV_MESSAGE "
|
1454
|
-
"*******************");
|
1455
|
-
return GRPC_CALL_ERROR;
|
1456
|
-
}
|
1457
|
-
|
1458
|
-
GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
|
1459
|
-
|
1460
|
-
if (nops == 0) {
|
1461
|
-
EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
|
1462
|
-
error = GRPC_CALL_OK;
|
1463
|
-
goto done;
|
1464
|
-
}
|
1465
|
-
|
1466
|
-
bctl = ReuseOrAllocateBatchControl(ops);
|
1467
|
-
if (bctl == nullptr) {
|
1468
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1469
|
-
}
|
1470
|
-
bctl->completion_data_.notify_tag.tag = notify_tag;
|
1471
|
-
bctl->completion_data_.notify_tag.is_closure =
|
1472
|
-
static_cast<uint8_t>(is_notify_tag_closure != 0);
|
1473
|
-
|
1474
|
-
stream_op = &bctl->op_;
|
1475
|
-
stream_op_payload = &stream_op_payload_;
|
1476
|
-
|
1477
|
-
// rewrite batch ops into a transport op
|
1478
|
-
for (i = 0; i < nops; i++) {
|
1479
|
-
op = &ops[i];
|
1480
|
-
if (op->reserved != nullptr) {
|
1481
|
-
error = GRPC_CALL_ERROR;
|
1482
|
-
goto done_with_error;
|
1483
|
-
}
|
1484
|
-
switch (op->op) {
|
1485
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
1486
|
-
// Flag validation: currently allow no flags
|
1487
|
-
if (!AreInitialMetadataFlagsValid(op->flags)) {
|
1488
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1489
|
-
goto done_with_error;
|
1490
|
-
}
|
1491
|
-
if (sent_initial_metadata_) {
|
1492
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1493
|
-
goto done_with_error;
|
1494
|
-
}
|
1495
|
-
if (op->data.send_initial_metadata.count > INT_MAX) {
|
1496
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1497
|
-
goto done_with_error;
|
1498
|
-
}
|
1499
|
-
stream_op->send_initial_metadata = true;
|
1500
|
-
sent_initial_metadata_ = true;
|
1501
|
-
if (!PrepareApplicationMetadata(op->data.send_initial_metadata.count,
|
1502
|
-
op->data.send_initial_metadata.metadata,
|
1503
|
-
false)) {
|
1504
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1505
|
-
goto done_with_error;
|
1506
|
-
}
|
1507
|
-
PrepareOutgoingInitialMetadata(*op, send_initial_metadata_);
|
1508
|
-
// TODO(ctiller): just make these the same variable?
|
1509
|
-
if (is_client() && send_deadline() != Timestamp::InfFuture()) {
|
1510
|
-
send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
|
1511
|
-
}
|
1512
|
-
if (is_client()) {
|
1513
|
-
send_initial_metadata_.Set(
|
1514
|
-
WaitForReady(),
|
1515
|
-
WaitForReady::ValueType{
|
1516
|
-
(op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
1517
|
-
(op->flags &
|
1518
|
-
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
1519
|
-
}
|
1520
|
-
stream_op_payload->send_initial_metadata.send_initial_metadata =
|
1521
|
-
&send_initial_metadata_;
|
1522
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1523
|
-
break;
|
1524
|
-
}
|
1525
|
-
case GRPC_OP_SEND_MESSAGE: {
|
1526
|
-
if (!AreWriteFlagsValid(op->flags)) {
|
1527
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1528
|
-
goto done_with_error;
|
1529
|
-
}
|
1530
|
-
if (op->data.send_message.send_message == nullptr) {
|
1531
|
-
error = GRPC_CALL_ERROR_INVALID_MESSAGE;
|
1532
|
-
goto done_with_error;
|
1533
|
-
}
|
1534
|
-
if (sending_message_) {
|
1535
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1536
|
-
goto done_with_error;
|
1537
|
-
}
|
1538
|
-
uint32_t flags = op->flags;
|
1539
|
-
// If the outgoing buffer is already compressed, mark it as so in the
|
1540
|
-
// flags. These will be picked up by the compression filter and further
|
1541
|
-
// (wasteful) attempts at compression skipped.
|
1542
|
-
if (op->data.send_message.send_message->data.raw.compression >
|
1543
|
-
GRPC_COMPRESS_NONE) {
|
1544
|
-
flags |= GRPC_WRITE_INTERNAL_COMPRESS;
|
1545
|
-
}
|
1546
|
-
stream_op->send_message = true;
|
1547
|
-
sending_message_ = true;
|
1548
|
-
send_slice_buffer_.Clear();
|
1549
|
-
grpc_slice_buffer_move_into(
|
1550
|
-
&op->data.send_message.send_message->data.raw.slice_buffer,
|
1551
|
-
send_slice_buffer_.c_slice_buffer());
|
1552
|
-
stream_op_payload->send_message.flags = flags;
|
1553
|
-
stream_op_payload->send_message.send_message = &send_slice_buffer_;
|
1554
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1555
|
-
break;
|
1556
|
-
}
|
1557
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
|
1558
|
-
// Flag validation: currently allow no flags
|
1559
|
-
if (op->flags != 0) {
|
1560
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1561
|
-
goto done_with_error;
|
1562
|
-
}
|
1563
|
-
if (!is_client()) {
|
1564
|
-
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1565
|
-
goto done_with_error;
|
1566
|
-
}
|
1567
|
-
if (sent_final_op_) {
|
1568
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1569
|
-
goto done_with_error;
|
1570
|
-
}
|
1571
|
-
stream_op->send_trailing_metadata = true;
|
1572
|
-
sent_final_op_ = true;
|
1573
|
-
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1574
|
-
&send_trailing_metadata_;
|
1575
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1576
|
-
break;
|
1577
|
-
}
|
1578
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
1579
|
-
// Flag validation: currently allow no flags
|
1580
|
-
if (op->flags != 0) {
|
1581
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1582
|
-
goto done_with_error;
|
1583
|
-
}
|
1584
|
-
if (is_client()) {
|
1585
|
-
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1586
|
-
goto done_with_error;
|
1587
|
-
}
|
1588
|
-
if (sent_final_op_) {
|
1589
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1590
|
-
goto done_with_error;
|
1591
|
-
}
|
1592
|
-
if (op->data.send_status_from_server.trailing_metadata_count >
|
1593
|
-
INT_MAX) {
|
1594
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1595
|
-
goto done_with_error;
|
1596
|
-
}
|
1597
|
-
stream_op->send_trailing_metadata = true;
|
1598
|
-
sent_final_op_ = true;
|
1599
|
-
|
1600
|
-
if (!PrepareApplicationMetadata(
|
1601
|
-
op->data.send_status_from_server.trailing_metadata_count,
|
1602
|
-
op->data.send_status_from_server.trailing_metadata, true)) {
|
1603
|
-
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1604
|
-
goto done_with_error;
|
1605
|
-
}
|
1606
|
-
|
1607
|
-
grpc_error_handle status_error =
|
1608
|
-
op->data.send_status_from_server.status == GRPC_STATUS_OK
|
1609
|
-
? absl::OkStatus()
|
1610
|
-
: grpc_error_set_int(
|
1611
|
-
GRPC_ERROR_CREATE("Server returned error"),
|
1612
|
-
StatusIntProperty::kRpcStatus,
|
1613
|
-
static_cast<intptr_t>(
|
1614
|
-
op->data.send_status_from_server.status));
|
1615
|
-
if (op->data.send_status_from_server.status_details != nullptr) {
|
1616
|
-
send_trailing_metadata_.Set(
|
1617
|
-
GrpcMessageMetadata(),
|
1618
|
-
Slice(grpc_slice_copy(
|
1619
|
-
*op->data.send_status_from_server.status_details)));
|
1620
|
-
if (!status_error.ok()) {
|
1621
|
-
status_error = grpc_error_set_str(
|
1622
|
-
status_error, StatusStrProperty::kGrpcMessage,
|
1623
|
-
StringViewFromSlice(
|
1624
|
-
*op->data.send_status_from_server.status_details));
|
1625
|
-
}
|
1626
|
-
}
|
1627
|
-
|
1628
|
-
status_error_.set(status_error);
|
1629
|
-
|
1630
|
-
send_trailing_metadata_.Set(GrpcStatusMetadata(),
|
1631
|
-
op->data.send_status_from_server.status);
|
1632
|
-
|
1633
|
-
// Ignore any te metadata key value pairs specified.
|
1634
|
-
send_trailing_metadata_.Remove(TeMetadata());
|
1635
|
-
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1636
|
-
&send_trailing_metadata_;
|
1637
|
-
stream_op_payload->send_trailing_metadata.sent =
|
1638
|
-
&sent_server_trailing_metadata_;
|
1639
|
-
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1640
|
-
break;
|
1641
|
-
}
|
1642
|
-
case GRPC_OP_RECV_INITIAL_METADATA: {
|
1643
|
-
// Flag validation: currently allow no flags
|
1644
|
-
if (op->flags != 0) {
|
1645
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1646
|
-
goto done_with_error;
|
1647
|
-
}
|
1648
|
-
if (received_initial_metadata_) {
|
1649
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1650
|
-
goto done_with_error;
|
1651
|
-
}
|
1652
|
-
received_initial_metadata_ = true;
|
1653
|
-
buffered_metadata_[0] =
|
1654
|
-
op->data.recv_initial_metadata.recv_initial_metadata;
|
1655
|
-
GRPC_CLOSURE_INIT(
|
1656
|
-
&receiving_initial_metadata_ready_,
|
1657
|
-
[](void* bctl, grpc_error_handle error) {
|
1658
|
-
static_cast<BatchControl*>(bctl)->ReceivingInitialMetadataReady(
|
1659
|
-
error);
|
1660
|
-
},
|
1661
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1662
|
-
stream_op->recv_initial_metadata = true;
|
1663
|
-
stream_op_payload->recv_initial_metadata.recv_initial_metadata =
|
1664
|
-
&recv_initial_metadata_;
|
1665
|
-
stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
|
1666
|
-
&receiving_initial_metadata_ready_;
|
1667
|
-
if (is_client()) {
|
1668
|
-
stream_op_payload->recv_initial_metadata.trailing_metadata_available =
|
1669
|
-
&is_trailers_only_;
|
1670
|
-
}
|
1671
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
|
1672
|
-
break;
|
1673
|
-
}
|
1674
|
-
case GRPC_OP_RECV_MESSAGE: {
|
1675
|
-
// Flag validation: currently allow no flags
|
1676
|
-
if (op->flags != 0) {
|
1677
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1678
|
-
goto done_with_error;
|
1679
|
-
}
|
1680
|
-
if (receiving_message_) {
|
1681
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1682
|
-
goto done_with_error;
|
1683
|
-
}
|
1684
|
-
receiving_message_ = true;
|
1685
|
-
stream_op->recv_message = true;
|
1686
|
-
receiving_slice_buffer_.reset();
|
1687
|
-
receiving_buffer_ = op->data.recv_message.recv_message;
|
1688
|
-
stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
|
1689
|
-
receiving_stream_flags_ = 0;
|
1690
|
-
stream_op_payload->recv_message.flags = &receiving_stream_flags_;
|
1691
|
-
stream_op_payload->recv_message.call_failed_before_recv_message =
|
1692
|
-
&call_failed_before_recv_message_;
|
1693
|
-
GRPC_CLOSURE_INIT(
|
1694
|
-
&receiving_stream_ready_,
|
1695
|
-
[](void* bctlp, grpc_error_handle error) {
|
1696
|
-
auto* bctl = static_cast<BatchControl*>(bctlp);
|
1697
|
-
auto* call = bctl->call_;
|
1698
|
-
// Yields the call combiner before processing the received
|
1699
|
-
// message.
|
1700
|
-
GRPC_CALL_COMBINER_STOP(call->call_combiner(),
|
1701
|
-
"recv_message_ready");
|
1702
|
-
bctl->ReceivingStreamReady(error);
|
1703
|
-
},
|
1704
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1705
|
-
stream_op_payload->recv_message.recv_message_ready =
|
1706
|
-
&receiving_stream_ready_;
|
1707
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
|
1708
|
-
break;
|
1709
|
-
}
|
1710
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
1711
|
-
// Flag validation: currently allow no flags
|
1712
|
-
if (op->flags != 0) {
|
1713
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1714
|
-
goto done_with_error;
|
1715
|
-
}
|
1716
|
-
if (!is_client()) {
|
1717
|
-
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1718
|
-
goto done_with_error;
|
1719
|
-
}
|
1720
|
-
if (requested_final_op_) {
|
1721
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1722
|
-
goto done_with_error;
|
1723
|
-
}
|
1724
|
-
requested_final_op_ = true;
|
1725
|
-
buffered_metadata_[1] =
|
1726
|
-
op->data.recv_status_on_client.trailing_metadata;
|
1727
|
-
final_op_.client.status = op->data.recv_status_on_client.status;
|
1728
|
-
final_op_.client.status_details =
|
1729
|
-
op->data.recv_status_on_client.status_details;
|
1730
|
-
final_op_.client.error_string =
|
1731
|
-
op->data.recv_status_on_client.error_string;
|
1732
|
-
stream_op->recv_trailing_metadata = true;
|
1733
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1734
|
-
&recv_trailing_metadata_;
|
1735
|
-
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1736
|
-
&final_info_.stats.transport_stream_stats;
|
1737
|
-
GRPC_CLOSURE_INIT(
|
1738
|
-
&receiving_trailing_metadata_ready_,
|
1739
|
-
[](void* bctl, grpc_error_handle error) {
|
1740
|
-
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1741
|
-
error);
|
1742
|
-
},
|
1743
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1744
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1745
|
-
&receiving_trailing_metadata_ready_;
|
1746
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1747
|
-
break;
|
1748
|
-
}
|
1749
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER: {
|
1750
|
-
// Flag validation: currently allow no flags
|
1751
|
-
if (op->flags != 0) {
|
1752
|
-
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1753
|
-
goto done_with_error;
|
1754
|
-
}
|
1755
|
-
if (is_client()) {
|
1756
|
-
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1757
|
-
goto done_with_error;
|
1758
|
-
}
|
1759
|
-
if (requested_final_op_) {
|
1760
|
-
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1761
|
-
goto done_with_error;
|
1762
|
-
}
|
1763
|
-
requested_final_op_ = true;
|
1764
|
-
final_op_.server.cancelled = op->data.recv_close_on_server.cancelled;
|
1765
|
-
stream_op->recv_trailing_metadata = true;
|
1766
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1767
|
-
&recv_trailing_metadata_;
|
1768
|
-
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1769
|
-
&final_info_.stats.transport_stream_stats;
|
1770
|
-
GRPC_CLOSURE_INIT(
|
1771
|
-
&receiving_trailing_metadata_ready_,
|
1772
|
-
[](void* bctl, grpc_error_handle error) {
|
1773
|
-
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1774
|
-
error);
|
1775
|
-
},
|
1776
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1777
|
-
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1778
|
-
&receiving_trailing_metadata_ready_;
|
1779
|
-
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1780
|
-
break;
|
1781
|
-
}
|
1782
|
-
}
|
1783
|
-
}
|
1784
|
-
|
1785
|
-
InternalRef("completion");
|
1786
|
-
if (!is_notify_tag_closure) {
|
1787
|
-
CHECK(grpc_cq_begin_op(cq_, notify_tag));
|
1788
|
-
}
|
1789
|
-
bctl->set_pending_ops(pending_ops);
|
1790
|
-
|
1791
|
-
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
1792
|
-
GRPC_CLOSURE_INIT(
|
1793
|
-
&bctl->finish_batch_,
|
1794
|
-
[](void* bctl, grpc_error_handle error) {
|
1795
|
-
static_cast<BatchControl*>(bctl)->FinishBatch(error);
|
1796
|
-
},
|
1797
|
-
bctl, grpc_schedule_on_exec_ctx);
|
1798
|
-
stream_op->on_complete = &bctl->finish_batch_;
|
1799
|
-
}
|
1800
|
-
|
1801
|
-
if (grpc_call_trace.enabled()) {
|
1802
|
-
gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1803
|
-
PendingOpString(pending_ops).c_str(),
|
1804
|
-
grpc_transport_stream_op_batch_string(stream_op, false).c_str(),
|
1805
|
-
bctl->completion_data_.notify_tag.tag);
|
1806
|
-
}
|
1807
|
-
ExecuteBatch(stream_op, &bctl->start_batch_);
|
1808
|
-
|
1809
|
-
done:
|
1810
|
-
return error;
|
1811
|
-
|
1812
|
-
done_with_error:
|
1813
|
-
// reverse any mutations that occurred
|
1814
|
-
if (stream_op->send_initial_metadata) {
|
1815
|
-
sent_initial_metadata_ = false;
|
1816
|
-
send_initial_metadata_.Clear();
|
1817
|
-
}
|
1818
|
-
if (stream_op->send_message) {
|
1819
|
-
sending_message_ = false;
|
1820
|
-
}
|
1821
|
-
if (stream_op->send_trailing_metadata) {
|
1822
|
-
sent_final_op_ = false;
|
1823
|
-
send_trailing_metadata_.Clear();
|
1824
|
-
}
|
1825
|
-
if (stream_op->recv_initial_metadata) {
|
1826
|
-
received_initial_metadata_ = false;
|
1827
|
-
}
|
1828
|
-
if (stream_op->recv_message) {
|
1829
|
-
receiving_message_ = false;
|
1830
|
-
}
|
1831
|
-
if (stream_op->recv_trailing_metadata) {
|
1832
|
-
requested_final_op_ = false;
|
1833
|
-
}
|
1834
|
-
goto done;
|
1835
|
-
}
|
1836
|
-
|
1837
|
-
void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
|
1838
|
-
void (*destroy)(void*)) {
|
1839
|
-
if (context_[elem].destroy) {
|
1840
|
-
context_[elem].destroy(context_[elem].value);
|
1841
|
-
}
|
1842
|
-
context_[elem].value = value;
|
1843
|
-
context_[elem].destroy = destroy;
|
1844
|
-
}
|
1845
|
-
|
1846
|
-
///////////////////////////////////////////////////////////////////////////////
|
1847
|
-
// Metadata validation helpers
|
1848
|
-
|
1849
|
-
namespace {
|
1850
|
-
bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
|
1851
|
-
if (count > INT_MAX) {
|
1852
|
-
return false;
|
1853
|
-
}
|
1854
|
-
for (size_t i = 0; i < count; i++) {
|
1855
|
-
grpc_metadata* md = &metadata[i];
|
1856
|
-
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
1857
|
-
grpc_validate_header_key_is_legal(md->key))) {
|
1858
|
-
return false;
|
1859
|
-
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1860
|
-
!GRPC_LOG_IF_ERROR(
|
1861
|
-
"validate_metadata",
|
1862
|
-
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1863
|
-
return false;
|
1864
|
-
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1865
|
-
// HTTP2 hpack encoding has a maximum limit.
|
1866
|
-
return false;
|
1867
|
-
}
|
1868
|
-
}
|
1869
|
-
return true;
|
1870
|
-
}
|
1871
|
-
} // namespace
|
1872
|
-
|
1873
|
-
///////////////////////////////////////////////////////////////////////////////
|
1874
|
-
// PromiseBasedCall
|
1875
|
-
// Will be folded into Call once the promise conversion is done
|
1876
|
-
|
1877
|
-
class BasicPromiseBasedCall : public Call, public Party {
|
1878
|
-
public:
|
1879
|
-
using Call::arena;
|
1880
|
-
|
1881
|
-
BasicPromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
1882
|
-
uint32_t initial_internal_refs,
|
1883
|
-
const grpc_call_create_args& args)
|
1884
|
-
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
1885
|
-
args.channel->Ref()),
|
1886
|
-
Party(initial_internal_refs),
|
1887
|
-
external_refs_(initial_external_refs),
|
1888
|
-
cq_(args.cq) {
|
1889
|
-
if (args.cq != nullptr) {
|
1890
|
-
GRPC_CQ_INTERNAL_REF(args.cq, "bind");
|
1891
|
-
}
|
1892
|
-
context_[GRPC_CONTEXT_CALL].value = this;
|
1893
|
-
}
|
1894
|
-
|
1895
|
-
~BasicPromiseBasedCall() override {
|
1896
|
-
if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
|
1897
|
-
for (int i = 0; i < GRPC_CONTEXT_COUNT; i++) {
|
1898
|
-
if (context_[i].destroy) {
|
1899
|
-
context_[i].destroy(context_[i].value);
|
1900
|
-
}
|
1901
|
-
}
|
1902
|
-
}
|
1903
|
-
|
1904
|
-
virtual void OrphanCall() = 0;
|
1905
|
-
|
1906
|
-
virtual ServerCallContext* server_call_context() { return nullptr; }
|
1907
|
-
void SetCompletionQueue(grpc_completion_queue* cq) final {
|
1908
|
-
cq_ = cq;
|
1909
|
-
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
1910
|
-
}
|
1911
|
-
|
1912
|
-
// Implementation of call refcounting: move this to DualRefCounted once we
|
1913
|
-
// don't need to maintain FilterStackCall compatibility
|
1914
|
-
void ExternalRef() final {
|
1915
|
-
if (external_refs_.fetch_add(1, std::memory_order_relaxed) == 0) {
|
1916
|
-
InternalRef("external");
|
1917
|
-
}
|
1918
|
-
}
|
1919
|
-
void ExternalUnref() final {
|
1920
|
-
if (external_refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
|
1921
|
-
OrphanCall();
|
1922
|
-
InternalUnref("external");
|
1923
|
-
}
|
1924
|
-
}
|
1925
|
-
void InternalRef(const char* reason) final {
|
1926
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1927
|
-
gpr_log(GPR_DEBUG, "INTERNAL_REF:%p:%s", this, reason);
|
1928
|
-
}
|
1929
|
-
Party::IncrementRefCount();
|
1930
|
-
}
|
1931
|
-
void InternalUnref(const char* reason) final {
|
1932
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1933
|
-
gpr_log(GPR_DEBUG, "INTERNAL_UNREF:%p:%s", this, reason);
|
1934
|
-
}
|
1935
|
-
Party::Unref();
|
1936
|
-
}
|
1937
|
-
|
1938
|
-
void RunInContext(absl::AnyInvocable<void()> fn) {
|
1939
|
-
Spawn(
|
1940
|
-
"run_in_context",
|
1941
|
-
[fn = std::move(fn)]() mutable {
|
1942
|
-
fn();
|
1943
|
-
return Empty{};
|
1944
|
-
},
|
1945
|
-
[](Empty) {});
|
1946
|
-
}
|
1947
|
-
|
1948
|
-
void ContextSet(grpc_context_index elem, void* value,
|
1949
|
-
void (*destroy)(void*)) final {
|
1950
|
-
if (context_[elem].destroy != nullptr) {
|
1951
|
-
context_[elem].destroy(context_[elem].value);
|
1952
|
-
}
|
1953
|
-
context_[elem].value = value;
|
1954
|
-
context_[elem].destroy = destroy;
|
1955
|
-
}
|
1956
|
-
|
1957
|
-
void* ContextGet(grpc_context_index elem) const final {
|
1958
|
-
return context_[elem].value;
|
1959
|
-
}
|
1960
|
-
|
1961
|
-
// Accept the stats from the context (call once we have proof the transport is
|
1962
|
-
// done with them).
|
1963
|
-
void AcceptTransportStatsFromContext() {
|
1964
|
-
final_stats_ = *call_context_.call_stats();
|
1965
|
-
}
|
1966
|
-
|
1967
|
-
// This should return nullptr for the promise stack (and alternative means
|
1968
|
-
// for that functionality be invented)
|
1969
|
-
grpc_call_stack* call_stack() final { return nullptr; }
|
1970
|
-
|
1971
|
-
virtual RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs) {
|
1972
|
-
Crash("Not implemented");
|
1973
|
-
}
|
1974
|
-
|
1975
|
-
protected:
|
1976
|
-
class ScopedContext
|
1977
|
-
: public ScopedActivity,
|
1978
|
-
public promise_detail::Context<Arena>,
|
1979
|
-
public promise_detail::Context<grpc_call_context_element>,
|
1980
|
-
public promise_detail::Context<CallContext>,
|
1981
|
-
public promise_detail::Context<CallFinalization> {
|
1982
|
-
public:
|
1983
|
-
explicit ScopedContext(BasicPromiseBasedCall* call)
|
1984
|
-
: ScopedActivity(call),
|
1985
|
-
promise_detail::Context<Arena>(call->arena()),
|
1986
|
-
promise_detail::Context<grpc_call_context_element>(call->context_),
|
1987
|
-
promise_detail::Context<CallContext>(&call->call_context_),
|
1988
|
-
promise_detail::Context<CallFinalization>(&call->finalization_) {}
|
1989
|
-
};
|
1990
|
-
|
1991
|
-
grpc_call_context_element* context() { return context_; }
|
1992
|
-
|
1993
|
-
grpc_completion_queue* cq() { return cq_; }
|
1994
|
-
|
1995
|
-
// At the end of the call run any finalization actions.
|
1996
|
-
void SetFinalizationStatus(grpc_status_code status, Slice status_details) {
|
1997
|
-
final_message_ = std::move(status_details);
|
1998
|
-
final_status_ = status;
|
1999
|
-
}
|
2000
|
-
|
2001
|
-
grpc_event_engine::experimental::EventEngine* event_engine() const override {
|
2002
|
-
return channel()->event_engine();
|
2003
|
-
}
|
2004
|
-
|
2005
|
-
private:
|
2006
|
-
void PartyOver() final {
|
2007
|
-
{
|
2008
|
-
ScopedContext ctx(this);
|
2009
|
-
std::string message;
|
2010
|
-
grpc_call_final_info final_info;
|
2011
|
-
final_info.stats = final_stats_;
|
2012
|
-
final_info.final_status = final_status_;
|
2013
|
-
// TODO(ctiller): change type here so we don't need to copy this string.
|
2014
|
-
final_info.error_string = nullptr;
|
2015
|
-
if (!final_message_.empty()) {
|
2016
|
-
message = std::string(final_message_.begin(), final_message_.end());
|
2017
|
-
final_info.error_string = message.c_str();
|
2018
|
-
}
|
2019
|
-
final_info.stats.latency =
|
2020
|
-
gpr_cycle_counter_sub(gpr_get_cycle_counter(), start_time());
|
2021
|
-
finalization_.Run(&final_info);
|
2022
|
-
CancelRemainingParticipants();
|
2023
|
-
arena()->DestroyManagedNewObjects();
|
2024
|
-
}
|
2025
|
-
DeleteThis();
|
2026
|
-
}
|
2027
|
-
|
2028
|
-
// Double refcounted for now: party owns the internal refcount, we track the
|
2029
|
-
// external refcount. Figure out a better scheme post-promise conversion.
|
2030
|
-
std::atomic<size_t> external_refs_;
|
2031
|
-
CallFinalization finalization_;
|
2032
|
-
CallContext call_context_{this};
|
2033
|
-
// Contexts for various subsystems (security, tracing, ...).
|
2034
|
-
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
2035
|
-
grpc_call_stats final_stats_{};
|
2036
|
-
Slice final_message_;
|
2037
|
-
grpc_status_code final_status_ = GRPC_STATUS_UNKNOWN;
|
2038
|
-
grpc_completion_queue* cq_;
|
2039
|
-
};
|
2040
|
-
|
2041
|
-
class PromiseBasedCall : public BasicPromiseBasedCall {
|
2042
|
-
public:
|
2043
|
-
PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2044
|
-
const grpc_call_create_args& args);
|
2045
|
-
|
2046
|
-
bool Completed() final { return finished_.IsSet(); }
|
2047
|
-
|
2048
|
-
bool failed_before_recv_message() const final {
|
2049
|
-
return failed_before_recv_message_.load(std::memory_order_relaxed);
|
2050
|
-
}
|
2051
|
-
|
2052
|
-
using Call::arena;
|
2053
|
-
|
2054
|
-
protected:
|
2055
|
-
class ScopedContext : public BasicPromiseBasedCall::ScopedContext,
|
2056
|
-
public BatchBuilder,
|
2057
|
-
public promise_detail::Context<BatchBuilder> {
|
2058
|
-
public:
|
2059
|
-
explicit ScopedContext(PromiseBasedCall* call)
|
2060
|
-
: BasicPromiseBasedCall::ScopedContext(call),
|
2061
|
-
BatchBuilder(&call->batch_payload_),
|
2062
|
-
promise_detail::Context<BatchBuilder>(this) {}
|
2063
|
-
};
|
2064
|
-
|
2065
|
-
class Completion {
|
2066
|
-
public:
|
2067
|
-
Completion() : index_(kNullIndex) {}
|
2068
|
-
~Completion() { CHECK(index_ == kNullIndex); }
|
2069
|
-
explicit Completion(uint8_t index) : index_(index) {}
|
2070
|
-
Completion(const Completion& other) = delete;
|
2071
|
-
Completion& operator=(const Completion& other) = delete;
|
2072
|
-
Completion(Completion&& other) noexcept : index_(other.index_) {
|
2073
|
-
other.index_ = kNullIndex;
|
2074
|
-
}
|
2075
|
-
Completion& operator=(Completion&& other) noexcept {
|
2076
|
-
CHECK(index_ == kNullIndex);
|
2077
|
-
index_ = other.index_;
|
2078
|
-
other.index_ = kNullIndex;
|
2079
|
-
return *this;
|
2080
|
-
}
|
2081
|
-
|
2082
|
-
uint8_t index() const { return index_; }
|
2083
|
-
uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
|
2084
|
-
bool has_value() const { return index_ != kNullIndex; }
|
2085
|
-
|
2086
|
-
private:
|
2087
|
-
enum : uint8_t { kNullIndex = 0xff };
|
2088
|
-
uint8_t index_;
|
2089
|
-
};
|
2090
|
-
|
2091
|
-
// Enumerates why a Completion is still pending
|
2092
|
-
enum class PendingOp {
|
2093
|
-
// We're in the midst of starting a batch of operations
|
2094
|
-
kStartingBatch = 0,
|
2095
|
-
// The following correspond with the batch operations from above
|
2096
|
-
kSendInitialMetadata,
|
2097
|
-
kReceiveInitialMetadata,
|
2098
|
-
kReceiveStatusOnClient,
|
2099
|
-
kReceiveCloseOnServer = kReceiveStatusOnClient,
|
2100
|
-
kSendMessage,
|
2101
|
-
kReceiveMessage,
|
2102
|
-
kSendStatusFromServer,
|
2103
|
-
kSendCloseFromClient = kSendStatusFromServer,
|
2104
|
-
};
|
2105
|
-
|
2106
|
-
bool RunParty() override {
|
2107
|
-
ScopedContext ctx(this);
|
2108
|
-
return Party::RunParty();
|
2109
|
-
}
|
2110
|
-
|
2111
|
-
const char* PendingOpString(PendingOp reason) const {
|
2112
|
-
switch (reason) {
|
2113
|
-
case PendingOp::kStartingBatch:
|
2114
|
-
return "StartingBatch";
|
2115
|
-
case PendingOp::kSendInitialMetadata:
|
2116
|
-
return "SendInitialMetadata";
|
2117
|
-
case PendingOp::kReceiveInitialMetadata:
|
2118
|
-
return "ReceiveInitialMetadata";
|
2119
|
-
case PendingOp::kReceiveStatusOnClient:
|
2120
|
-
return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
|
2121
|
-
case PendingOp::kSendMessage:
|
2122
|
-
return "SendMessage";
|
2123
|
-
case PendingOp::kReceiveMessage:
|
2124
|
-
return "ReceiveMessage";
|
2125
|
-
case PendingOp::kSendStatusFromServer:
|
2126
|
-
return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
|
2127
|
-
}
|
2128
|
-
return "Unknown";
|
2129
|
-
}
|
2130
|
-
|
2131
|
-
static constexpr uint32_t PendingOpBit(PendingOp reason) {
|
2132
|
-
return 1 << static_cast<int>(reason);
|
2133
|
-
}
|
2134
|
-
|
2135
|
-
// Begin work on a completion, recording the tag/closure to notify.
|
2136
|
-
// Use the op selected in \a ops to determine the index to allocate into.
|
2137
|
-
// Starts the "StartingBatch" PendingOp immediately.
|
2138
|
-
// Assumes at least one operation in \a ops.
|
2139
|
-
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops);
|
2140
|
-
// Add one pending op to the completion, and return it.
|
2141
|
-
Completion AddOpToCompletion(const Completion& completion, PendingOp reason);
|
2142
|
-
// Stringify a completion
|
2143
|
-
std::string CompletionString(const Completion& completion) const {
|
2144
|
-
return completion.has_value()
|
2145
|
-
? completion_info_[completion.index()].pending.ToString(this)
|
2146
|
-
: "no-completion";
|
2147
|
-
}
|
2148
|
-
// Finish one op on the completion. Must have been previously been added.
|
2149
|
-
// The completion as a whole finishes when all pending ops finish.
|
2150
|
-
void FinishOpOnCompletion(Completion* completion, PendingOp reason);
|
2151
|
-
// Mark the completion as failed. Does not finish it.
|
2152
|
-
void FailCompletion(const Completion& completion,
|
2153
|
-
SourceLocation source_location = {});
|
2154
|
-
// Mark the completion as infallible. Overrides FailCompletion to report
|
2155
|
-
// success always.
|
2156
|
-
void ForceCompletionSuccess(const Completion& completion);
|
2157
|
-
|
2158
|
-
std::string PresentAndCompletionText(const char* caption, bool has,
|
2159
|
-
const Completion& completion) const {
|
2160
|
-
if (has) {
|
2161
|
-
if (completion.has_value()) {
|
2162
|
-
return absl::StrCat(caption, ":", CompletionString(completion), " ");
|
2163
|
-
} else {
|
2164
|
-
return absl::StrCat(caption,
|
2165
|
-
":!!BUG:operation is present, no completion!! ");
|
2166
|
-
}
|
2167
|
-
} else {
|
2168
|
-
if (!completion.has_value()) {
|
2169
|
-
return "";
|
2170
|
-
} else {
|
2171
|
-
return absl::StrCat(caption, ":no-op:", CompletionString(completion),
|
2172
|
-
" ");
|
2173
|
-
}
|
2174
|
-
}
|
2175
|
-
}
|
2176
|
-
|
2177
|
-
// Spawn a job that will first do FirstPromise then receive a message
|
2178
|
-
template <typename FirstPromise>
|
2179
|
-
void StartRecvMessage(const grpc_op& op, const Completion& completion,
|
2180
|
-
FirstPromise first,
|
2181
|
-
PipeReceiver<MessageHandle>* receiver,
|
2182
|
-
bool cancel_on_error, Party::BulkSpawner& spawner);
|
2183
|
-
void StartSendMessage(const grpc_op& op, const Completion& completion,
|
2184
|
-
PipeSender<MessageHandle>* sender,
|
2185
|
-
Party::BulkSpawner& spawner);
|
2186
|
-
|
2187
|
-
void set_completed() { finished_.Set(); }
|
2188
|
-
|
2189
|
-
// Returns a promise that resolves to Empty whenever the call is completed.
|
2190
|
-
auto finished() { return finished_.Wait(); }
|
2191
|
-
|
2192
|
-
// Returns a promise that resolves to Empty whenever there is no outstanding
|
2193
|
-
// send operation
|
2194
|
-
auto WaitForSendingStarted() {
|
2195
|
-
return [this]() -> Poll<Empty> {
|
2196
|
-
int n = sends_queued_.load(std::memory_order_relaxed);
|
2197
|
-
if (grpc_call_trace.enabled()) {
|
2198
|
-
gpr_log(GPR_DEBUG, "%s[call] WaitForSendingStarted n=%d",
|
2199
|
-
DebugTag().c_str(), n);
|
2200
|
-
}
|
2201
|
-
if (n != 0) return waiting_for_queued_sends_.pending();
|
2202
|
-
return Empty{};
|
2203
|
-
};
|
2204
|
-
}
|
2205
|
-
|
2206
|
-
// Mark that a send has been queued - blocks sending trailing metadata.
|
2207
|
-
void QueueSend() {
|
2208
|
-
if (grpc_call_trace.enabled()) {
|
2209
|
-
gpr_log(GPR_DEBUG, "%s[call] QueueSend", DebugTag().c_str());
|
2210
|
-
}
|
2211
|
-
sends_queued_.fetch_add(1, std::memory_order_relaxed);
|
2212
|
-
}
|
2213
|
-
// Mark that a send has been dequeued - allows sending trailing metadata once
|
2214
|
-
// zero sends are queued.
|
2215
|
-
void EnactSend() {
|
2216
|
-
if (grpc_call_trace.enabled()) {
|
2217
|
-
gpr_log(GPR_DEBUG, "%s[call] EnactSend", DebugTag().c_str());
|
2218
|
-
}
|
2219
|
-
if (1 == sends_queued_.fetch_sub(1, std::memory_order_relaxed)) {
|
2220
|
-
waiting_for_queued_sends_.Wake();
|
2221
|
-
}
|
2222
|
-
}
|
2223
|
-
|
2224
|
-
void set_failed_before_recv_message() {
|
2225
|
-
failed_before_recv_message_.store(true, std::memory_order_relaxed);
|
2226
|
-
}
|
2227
|
-
|
2228
|
-
private:
|
2229
|
-
union CompletionInfo {
|
2230
|
-
static constexpr uint32_t kOpFailed = 0x8000'0000u;
|
2231
|
-
static constexpr uint32_t kOpForceSuccess = 0x4000'0000u;
|
2232
|
-
CompletionInfo() {}
|
2233
|
-
enum CompletionState {
|
2234
|
-
kPending,
|
2235
|
-
kSuccess,
|
2236
|
-
kFailure,
|
2237
|
-
};
|
2238
|
-
struct Pending {
|
2239
|
-
// Bitmask of PendingOps at the bottom, and kOpFailed, kOpForceSuccess at
|
2240
|
-
// the top.
|
2241
|
-
std::atomic<uint32_t> state;
|
2242
|
-
bool is_closure;
|
2243
|
-
// True if this completion was for a recv_message op.
|
2244
|
-
// In that case if the completion as a whole fails we need to cleanup the
|
2245
|
-
// returned message.
|
2246
|
-
bool is_recv_message;
|
2247
|
-
void* tag;
|
2248
|
-
|
2249
|
-
void Start(bool is_closure, void* tag) {
|
2250
|
-
this->is_closure = is_closure;
|
2251
|
-
this->is_recv_message = false;
|
2252
|
-
this->tag = tag;
|
2253
|
-
state.store(PendingOpBit(PendingOp::kStartingBatch),
|
2254
|
-
std::memory_order_release);
|
2255
|
-
}
|
2256
|
-
|
2257
|
-
void AddPendingBit(PendingOp reason) {
|
2258
|
-
if (reason == PendingOp::kReceiveMessage) is_recv_message = true;
|
2259
|
-
auto prev =
|
2260
|
-
state.fetch_or(PendingOpBit(reason), std::memory_order_relaxed);
|
2261
|
-
CHECK_EQ((prev & PendingOpBit(reason)), 0u);
|
2262
|
-
}
|
2263
|
-
|
2264
|
-
CompletionState RemovePendingBit(PendingOp reason) {
|
2265
|
-
const uint32_t mask = ~PendingOpBit(reason);
|
2266
|
-
auto prev = state.fetch_and(mask, std::memory_order_acq_rel);
|
2267
|
-
CHECK_NE((prev & PendingOpBit(reason)), 0u);
|
2268
|
-
switch (prev & mask) {
|
2269
|
-
case kOpFailed:
|
2270
|
-
return kFailure;
|
2271
|
-
case kOpFailed | kOpForceSuccess:
|
2272
|
-
case kOpForceSuccess:
|
2273
|
-
case 0:
|
2274
|
-
return kSuccess;
|
2275
|
-
default:
|
2276
|
-
return kPending;
|
2277
|
-
}
|
2278
|
-
}
|
2279
|
-
|
2280
|
-
void MarkFailed() {
|
2281
|
-
state.fetch_or(kOpFailed, std::memory_order_relaxed);
|
2282
|
-
}
|
2283
|
-
|
2284
|
-
void MarkForceSuccess() {
|
2285
|
-
state.fetch_or(kOpForceSuccess, std::memory_order_relaxed);
|
2286
|
-
}
|
2287
|
-
|
2288
|
-
std::string ToString(const PromiseBasedCall* call) const {
|
2289
|
-
auto state = this->state.load(std::memory_order_relaxed);
|
2290
|
-
std::vector<absl::string_view> pending_ops;
|
2291
|
-
for (size_t i = 0; i < 24; i++) {
|
2292
|
-
if (state & (1u << i)) {
|
2293
|
-
pending_ops.push_back(
|
2294
|
-
call->PendingOpString(static_cast<PendingOp>(i)));
|
2295
|
-
}
|
2296
|
-
}
|
2297
|
-
return absl::StrFormat("{%s}%s:tag=%p", absl::StrJoin(pending_ops, ","),
|
2298
|
-
(state & kOpForceSuccess) ? ":force-success"
|
2299
|
-
: (state & kOpFailed) ? ":failed"
|
2300
|
-
: ":success",
|
2301
|
-
tag);
|
2302
|
-
}
|
2303
|
-
} pending;
|
2304
|
-
grpc_cq_completion completion;
|
2305
|
-
};
|
2306
|
-
|
2307
|
-
CompletionInfo completion_info_[6];
|
2308
|
-
ExternallyObservableLatch<void> finished_;
|
2309
|
-
// Non-zero with an outstanding GRPC_OP_SEND_INITIAL_METADATA or
|
2310
|
-
// GRPC_OP_SEND_MESSAGE (one count each), and 0 once those payloads have been
|
2311
|
-
// pushed onto the outgoing pipe.
|
2312
|
-
std::atomic<uint8_t> sends_queued_{0};
|
2313
|
-
std::atomic<bool> failed_before_recv_message_{false};
|
2314
|
-
// Waiter for when sends_queued_ becomes 0.
|
2315
|
-
IntraActivityWaiter waiting_for_queued_sends_;
|
2316
|
-
grpc_byte_buffer** recv_message_ = nullptr;
|
2317
|
-
grpc_transport_stream_op_batch_payload batch_payload_{context()};
|
2318
|
-
};
|
2319
|
-
|
2320
|
-
template <typename T>
|
2321
|
-
grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
|
2322
|
-
grpc_call** out_call) {
|
2323
|
-
Channel* channel = args->channel.get();
|
2324
|
-
|
2325
|
-
auto* arena = channel->CreateArena();
|
2326
|
-
PromiseBasedCall* call = arena->New<T>(arena, args);
|
2327
|
-
*out_call = call->c_ptr();
|
2328
|
-
DCHECK(Call::FromC(*out_call) == call);
|
2329
|
-
return absl::OkStatus();
|
2330
|
-
}
|
2331
|
-
|
2332
|
-
PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2333
|
-
const grpc_call_create_args& args)
|
2334
|
-
: BasicPromiseBasedCall(arena, initial_external_refs,
|
2335
|
-
initial_external_refs != 0 ? 1 : 0, args) {}
|
2336
|
-
|
2337
|
-
static void CToMetadata(grpc_metadata* metadata, size_t count,
|
2338
|
-
grpc_metadata_batch* b) {
|
2339
|
-
for (size_t i = 0; i < count; i++) {
|
2340
|
-
grpc_metadata* md = &metadata[i];
|
2341
|
-
auto key = StringViewFromSlice(md->key);
|
2342
|
-
// Filter "content-length metadata"
|
2343
|
-
if (key == "content-length") continue;
|
2344
|
-
b->Append(key, Slice(CSliceRef(md->value)),
|
2345
|
-
[md](absl::string_view error, const Slice& value) {
|
2346
|
-
gpr_log(GPR_DEBUG, "Append error: %s",
|
2347
|
-
absl::StrCat("key=", StringViewFromSlice(md->key),
|
2348
|
-
" error=", error,
|
2349
|
-
" value=", value.as_string_view())
|
2350
|
-
.c_str());
|
2351
|
-
});
|
2352
|
-
}
|
2353
|
-
}
|
2354
|
-
|
2355
|
-
PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
|
2356
|
-
void* tag, bool is_closure, const grpc_op* ops) {
|
2357
|
-
Completion c(BatchSlotForOp(ops[0].op));
|
2358
|
-
if (!is_closure) {
|
2359
|
-
grpc_cq_begin_op(cq(), tag);
|
2360
|
-
}
|
2361
|
-
completion_info_[c.index()].pending.Start(is_closure, tag);
|
2362
|
-
if (grpc_call_trace.enabled()) {
|
2363
|
-
gpr_log(GPR_INFO, "%s[call] StartCompletion %s", DebugTag().c_str(),
|
2364
|
-
CompletionString(c).c_str());
|
2365
|
-
}
|
2366
|
-
return c;
|
2367
|
-
}
|
2368
|
-
|
2369
|
-
PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
|
2370
|
-
const Completion& completion, PendingOp reason) {
|
2371
|
-
if (grpc_call_trace.enabled()) {
|
2372
|
-
gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
|
2373
|
-
CompletionString(completion).c_str(), PendingOpString(reason));
|
2374
|
-
}
|
2375
|
-
CHECK(completion.has_value());
|
2376
|
-
completion_info_[completion.index()].pending.AddPendingBit(reason);
|
2377
|
-
return Completion(completion.index());
|
2378
|
-
}
|
2379
|
-
|
2380
|
-
void PromiseBasedCall::FailCompletion(const Completion& completion,
|
2381
|
-
SourceLocation location) {
|
2382
|
-
if (grpc_call_trace.enabled()) {
|
2383
|
-
gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
|
2384
|
-
"%s[call] FailCompletion %s", DebugTag().c_str(),
|
2385
|
-
CompletionString(completion).c_str());
|
2386
|
-
}
|
2387
|
-
completion_info_[completion.index()].pending.MarkFailed();
|
2388
|
-
}
|
2389
|
-
|
2390
|
-
void PromiseBasedCall::ForceCompletionSuccess(const Completion& completion) {
|
2391
|
-
completion_info_[completion.index()].pending.MarkForceSuccess();
|
2392
|
-
}
|
2393
|
-
|
2394
|
-
void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
|
2395
|
-
PendingOp reason) {
|
2396
|
-
if (grpc_call_trace.enabled()) {
|
2397
|
-
gpr_log(GPR_INFO, "%s[call] FinishOpOnCompletion completion:%s finish:%s",
|
2398
|
-
DebugTag().c_str(), CompletionString(*completion).c_str(),
|
2399
|
-
PendingOpString(reason));
|
2400
|
-
}
|
2401
|
-
const uint8_t i = completion->TakeIndex();
|
2402
|
-
CHECK(i < GPR_ARRAY_SIZE(completion_info_));
|
2403
|
-
CompletionInfo::Pending& pending = completion_info_[i].pending;
|
2404
|
-
bool success;
|
2405
|
-
switch (pending.RemovePendingBit(reason)) {
|
2406
|
-
case CompletionInfo::kPending:
|
2407
|
-
return; // Early out
|
2408
|
-
case CompletionInfo::kSuccess:
|
2409
|
-
success = true;
|
2410
|
-
break;
|
2411
|
-
case CompletionInfo::kFailure:
|
2412
|
-
success = false;
|
2413
|
-
break;
|
2414
|
-
}
|
2415
|
-
if (pending.is_recv_message && !success && *recv_message_ != nullptr) {
|
2416
|
-
grpc_byte_buffer_destroy(*recv_message_);
|
2417
|
-
*recv_message_ = nullptr;
|
2418
|
-
}
|
2419
|
-
auto error = success ? absl::OkStatus() : absl::CancelledError();
|
2420
|
-
if (pending.is_closure) {
|
2421
|
-
ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
|
2422
|
-
error);
|
2423
|
-
} else {
|
2424
|
-
InternalRef("cq_end_op");
|
2425
|
-
grpc_cq_end_op(
|
2426
|
-
cq(), pending.tag, error,
|
2427
|
-
[](void* p, grpc_cq_completion*) {
|
2428
|
-
static_cast<PromiseBasedCall*>(p)->InternalUnref("cq_end_op");
|
2429
|
-
},
|
2430
|
-
this, &completion_info_[i].completion);
|
2431
|
-
}
|
2432
|
-
}
|
2433
|
-
|
2434
|
-
void PromiseBasedCall::StartSendMessage(const grpc_op& op,
|
2435
|
-
const Completion& completion,
|
2436
|
-
PipeSender<MessageHandle>* sender,
|
2437
|
-
Party::BulkSpawner& spawner) {
|
2438
|
-
QueueSend();
|
2439
|
-
SliceBuffer send;
|
2440
|
-
grpc_slice_buffer_swap(
|
2441
|
-
&op.data.send_message.send_message->data.raw.slice_buffer,
|
2442
|
-
send.c_slice_buffer());
|
2443
|
-
auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
|
2444
|
-
spawner.Spawn(
|
2445
|
-
"call_send_message",
|
2446
|
-
[this, sender, msg = std::move(msg)]() mutable {
|
2447
|
-
EnactSend();
|
2448
|
-
return sender->Push(std::move(msg));
|
2449
|
-
},
|
2450
|
-
[this, completion = AddOpToCompletion(
|
2451
|
-
completion, PendingOp::kSendMessage)](bool result) mutable {
|
2452
|
-
if (grpc_call_trace.enabled()) {
|
2453
|
-
gpr_log(GPR_DEBUG, "%sSendMessage completes %s", DebugTag().c_str(),
|
2454
|
-
result ? "successfully" : "with failure");
|
2455
|
-
}
|
2456
|
-
if (!result) FailCompletion(completion);
|
2457
|
-
FinishOpOnCompletion(&completion, PendingOp::kSendMessage);
|
2458
|
-
});
|
2459
|
-
}
|
2460
|
-
|
2461
|
-
template <typename FirstPromiseFactory>
|
2462
|
-
void PromiseBasedCall::StartRecvMessage(
|
2463
|
-
const grpc_op& op, const Completion& completion,
|
2464
|
-
FirstPromiseFactory first_promise_factory,
|
2465
|
-
PipeReceiver<MessageHandle>* receiver, bool cancel_on_error,
|
2466
|
-
Party::BulkSpawner& spawner) {
|
2467
|
-
if (grpc_call_trace.enabled()) {
|
2468
|
-
gpr_log(GPR_INFO, "%s[call] Start RecvMessage: %s", DebugTag().c_str(),
|
2469
|
-
CompletionString(completion).c_str());
|
2470
|
-
}
|
2471
|
-
recv_message_ = op.data.recv_message.recv_message;
|
2472
|
-
spawner.Spawn(
|
2473
|
-
"call_recv_message",
|
2474
|
-
[first_promise_factory = std::move(first_promise_factory), receiver]() {
|
2475
|
-
return Seq(first_promise_factory(), receiver->Next());
|
2476
|
-
},
|
2477
|
-
[this, cancel_on_error,
|
2478
|
-
completion = AddOpToCompletion(completion, PendingOp::kReceiveMessage)](
|
2479
|
-
NextResult<MessageHandle> result) mutable {
|
2480
|
-
if (result.has_value()) {
|
2481
|
-
MessageHandle& message = *result;
|
2482
|
-
NoteLastMessageFlags(message->flags());
|
2483
|
-
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
2484
|
-
(incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
2485
|
-
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
2486
|
-
nullptr, 0, incoming_compression_algorithm());
|
2487
|
-
} else {
|
2488
|
-
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
2489
|
-
}
|
2490
|
-
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
2491
|
-
&(*recv_message_)->data.raw.slice_buffer);
|
2492
|
-
if (grpc_call_trace.enabled()) {
|
2493
|
-
gpr_log(GPR_INFO,
|
2494
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2495
|
-
"finishes: received %" PRIdPTR " byte message",
|
2496
|
-
DebugTag().c_str(),
|
2497
|
-
(*recv_message_)->data.raw.slice_buffer.length);
|
2498
|
-
}
|
2499
|
-
} else if (result.cancelled()) {
|
2500
|
-
if (grpc_call_trace.enabled()) {
|
2501
|
-
gpr_log(GPR_INFO,
|
2502
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2503
|
-
"finishes: received end-of-stream with error",
|
2504
|
-
DebugTag().c_str());
|
2505
|
-
}
|
2506
|
-
set_failed_before_recv_message();
|
2507
|
-
FailCompletion(completion);
|
2508
|
-
if (cancel_on_error) CancelWithError(absl::CancelledError());
|
2509
|
-
*recv_message_ = nullptr;
|
2510
|
-
} else {
|
2511
|
-
if (grpc_call_trace.enabled()) {
|
2512
|
-
gpr_log(GPR_INFO,
|
2513
|
-
"%s[call] RecvMessage: outstanding_recv "
|
2514
|
-
"finishes: received end-of-stream",
|
2515
|
-
DebugTag().c_str());
|
2516
|
-
}
|
2517
|
-
*recv_message_ = nullptr;
|
2518
|
-
}
|
2519
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveMessage);
|
2520
|
-
});
|
2521
|
-
}
|
2522
|
-
|
2523
|
-
///////////////////////////////////////////////////////////////////////////////
|
2524
|
-
// CallContext
|
2525
|
-
|
2526
|
-
void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
|
2527
|
-
call_->RunInContext(std::move(fn));
|
2528
|
-
}
|
2529
|
-
|
2530
|
-
void CallContext::IncrementRefCount(const char* reason) {
|
2531
|
-
call_->InternalRef(reason);
|
2532
|
-
}
|
2533
|
-
|
2534
|
-
void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
|
2535
|
-
|
2536
|
-
ServerCallContext* CallContext::server_call_context() {
|
2537
|
-
return call_->server_call_context();
|
2538
|
-
}
|
2539
|
-
|
2540
|
-
RefCountedPtr<CallSpineInterface> CallContext::MakeCallSpine(
|
2541
|
-
CallArgs call_args) {
|
2542
|
-
return call_->MakeCallSpine(std::move(call_args));
|
2543
|
-
}
|
2544
|
-
|
2545
|
-
grpc_call* CallContext::c_call() { return call_->c_ptr(); }
|
2546
|
-
|
2547
|
-
///////////////////////////////////////////////////////////////////////////////
|
2548
|
-
// PublishMetadataArray
|
2549
|
-
|
2550
|
-
namespace {
|
2551
|
-
void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array,
|
2552
|
-
bool is_client) {
|
2553
|
-
const auto md_count = md->count();
|
2554
|
-
if (md_count > array->capacity) {
|
2555
|
-
array->capacity =
|
2556
|
-
std::max(array->capacity + md->count(), array->capacity * 3 / 2);
|
2557
|
-
array->metadata = static_cast<grpc_metadata*>(
|
2558
|
-
gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
|
2559
|
-
}
|
2560
|
-
PublishToAppEncoder encoder(array, md, is_client);
|
2561
|
-
md->Encode(&encoder);
|
2562
|
-
}
|
2563
|
-
} // namespace
|
2564
|
-
|
2565
|
-
///////////////////////////////////////////////////////////////////////////////
|
2566
|
-
// ClientPromiseBasedCall
|
2567
|
-
|
2568
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
2569
|
-
class ClientPromiseBasedCall final : public PromiseBasedCall {
|
2570
|
-
public:
|
2571
|
-
ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
|
2572
|
-
: PromiseBasedCall(arena, 1, *args),
|
2573
|
-
polling_entity_(
|
2574
|
-
args->cq != nullptr
|
2575
|
-
? grpc_polling_entity_create_from_pollset(
|
2576
|
-
grpc_cq_pollset(args->cq))
|
2577
|
-
: (args->pollset_set_alternative != nullptr
|
2578
|
-
? grpc_polling_entity_create_from_pollset_set(
|
2579
|
-
args->pollset_set_alternative)
|
2580
|
-
: grpc_polling_entity{})) {
|
2581
|
-
global_stats().IncrementClientCallsCreated();
|
2582
|
-
if (args->cq != nullptr) {
|
2583
|
-
CHECK(args->pollset_set_alternative == nullptr)
|
2584
|
-
<< "Only one of 'cq' and 'pollset_set_alternative' should be "
|
2585
|
-
"non-nullptr.";
|
2586
|
-
}
|
2587
|
-
ScopedContext context(this);
|
2588
|
-
args->channel->channel_stack()->stats_plugin_group->AddClientCallTracers(
|
2589
|
-
*args->path, args->registered_method, this->context());
|
2590
|
-
send_initial_metadata_ = Arena::MakePooled<ClientMetadata>();
|
2591
|
-
send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
|
2592
|
-
if (args->authority.has_value()) {
|
2593
|
-
send_initial_metadata_->Set(HttpAuthorityMetadata(),
|
2594
|
-
std::move(*args->authority));
|
2595
|
-
}
|
2596
|
-
send_initial_metadata_->Set(GrpcRegisteredMethod(),
|
2597
|
-
reinterpret_cast<void*>(static_cast<uintptr_t>(
|
2598
|
-
args->registered_method)));
|
2599
|
-
if (auto* channelz_channel = channel()->channelz_node()) {
|
2600
|
-
channelz_channel->RecordCallStarted();
|
2601
|
-
}
|
2602
|
-
if (args->send_deadline != Timestamp::InfFuture()) {
|
2603
|
-
UpdateDeadline(args->send_deadline);
|
2604
|
-
}
|
2605
|
-
Call* parent = Call::FromC(args->parent);
|
2606
|
-
if (parent != nullptr) {
|
2607
|
-
auto parent_status = InitParent(parent, args->propagation_mask);
|
2608
|
-
if (!parent_status.ok()) {
|
2609
|
-
CancelWithError(std::move(parent_status));
|
2610
|
-
}
|
2611
|
-
PublishToParent(parent);
|
2612
|
-
}
|
2613
|
-
}
|
2614
|
-
|
2615
|
-
void OrphanCall() override { MaybeUnpublishFromParent(); }
|
2616
|
-
|
2617
|
-
~ClientPromiseBasedCall() override {
|
2618
|
-
ScopedContext context(this);
|
2619
|
-
send_initial_metadata_.reset();
|
2620
|
-
// Need to destroy the pipes under the ScopedContext above, so we
|
2621
|
-
// move them out here and then allow the destructors to run at
|
2622
|
-
// end of scope, but before context.
|
2623
|
-
auto c2s = std::move(client_to_server_messages_);
|
2624
|
-
auto s2c = std::move(server_to_client_messages_);
|
2625
|
-
auto sim = std::move(server_initial_metadata_);
|
2626
|
-
}
|
2627
|
-
|
2628
|
-
void CancelWithError(absl::Status error) override {
|
2629
|
-
if (cancel_with_error_called_.exchange(true, std::memory_order_relaxed)) {
|
2630
|
-
return;
|
2631
|
-
}
|
2632
|
-
if (!started_.exchange(true, std::memory_order_relaxed)) {
|
2633
|
-
// Initial metadata not sent yet, so we can just fail the call.
|
2634
|
-
Spawn(
|
2635
|
-
"cancel_before_initial_metadata",
|
2636
|
-
[error = std::move(error), this]() {
|
2637
|
-
server_to_client_messages_.sender.Close();
|
2638
|
-
auto md = ServerMetadataFromStatus(error);
|
2639
|
-
md->Set(GrpcCallWasCancelled(), true);
|
2640
|
-
Finish(std::move(md));
|
2641
|
-
return Empty{};
|
2642
|
-
},
|
2643
|
-
[](Empty) {});
|
2644
|
-
} else {
|
2645
|
-
Spawn(
|
2646
|
-
"cancel_with_error",
|
2647
|
-
[error = std::move(error), this]() {
|
2648
|
-
if (!cancel_error_.is_set()) {
|
2649
|
-
auto md = ServerMetadataFromStatus(error);
|
2650
|
-
md->Set(GrpcCallWasCancelled(), true);
|
2651
|
-
cancel_error_.Set(std::move(md));
|
2652
|
-
}
|
2653
|
-
return Empty{};
|
2654
|
-
},
|
2655
|
-
[](Empty) {});
|
2656
|
-
}
|
2657
|
-
}
|
2658
|
-
absl::string_view GetServerAuthority() const override { abort(); }
|
2659
|
-
bool is_trailers_only() const override { return is_trailers_only_; }
|
2660
|
-
|
2661
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
2662
|
-
bool is_notify_tag_closure) override;
|
2663
|
-
|
2664
|
-
std::string DebugTag() const override {
|
2665
|
-
return absl::StrFormat("CLIENT_CALL[%p]: ", this);
|
2666
|
-
}
|
2667
|
-
|
2668
|
-
RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs call_args) final {
|
2669
|
-
class WrappingCallSpine final : public PipeBasedCallSpine {
|
2670
|
-
public:
|
2671
|
-
WrappingCallSpine(ClientPromiseBasedCall* call,
|
2672
|
-
ClientMetadataHandle metadata)
|
2673
|
-
: call_(call) {
|
2674
|
-
call_->InternalRef("call-spine");
|
2675
|
-
SpawnInfallible(
|
2676
|
-
"send_client_initial_metadata",
|
2677
|
-
[self = Ref(), metadata = std::move(metadata)]() mutable {
|
2678
|
-
return Map(self->client_initial_metadata_.sender.Push(
|
2679
|
-
std::move(metadata)),
|
2680
|
-
[self](bool) { return Empty{}; });
|
2681
|
-
});
|
2682
|
-
}
|
2683
|
-
|
2684
|
-
~WrappingCallSpine() override {
|
2685
|
-
{
|
2686
|
-
ScopedContext context(call_);
|
2687
|
-
// Move these out and destroy before the internal unref.
|
2688
|
-
auto client_initial_metadata = std::move(client_initial_metadata_);
|
2689
|
-
auto server_trailing_metadata = std::move(server_trailing_metadata_);
|
2690
|
-
}
|
2691
|
-
call_->InternalUnref("call-spine");
|
2692
|
-
}
|
2693
|
-
|
2694
|
-
Pipe<ClientMetadataHandle>& client_initial_metadata() override {
|
2695
|
-
return client_initial_metadata_;
|
2696
|
-
}
|
2697
|
-
|
2698
|
-
Pipe<MessageHandle>& client_to_server_messages() override {
|
2699
|
-
return call_->client_to_server_messages_;
|
2700
|
-
}
|
2701
|
-
|
2702
|
-
Pipe<ServerMetadataHandle>& server_initial_metadata() override {
|
2703
|
-
return call_->server_initial_metadata_;
|
2704
|
-
}
|
2705
|
-
|
2706
|
-
Pipe<MessageHandle>& server_to_client_messages() override {
|
2707
|
-
return call_->server_to_client_messages_;
|
2708
|
-
}
|
2709
|
-
|
2710
|
-
Latch<ServerMetadataHandle>& cancel_latch() override {
|
2711
|
-
return cancel_error_;
|
2712
|
-
}
|
2713
|
-
|
2714
|
-
Latch<bool>& was_cancelled_latch() override {
|
2715
|
-
return was_cancelled_latch_;
|
2716
|
-
}
|
2717
|
-
|
2718
|
-
Party& party() override { return *call_; }
|
2719
|
-
Arena* arena() override { return call_->arena(); }
|
2720
|
-
|
2721
|
-
void IncrementRefCount() override { refs_.Ref(); }
|
2722
|
-
void Unref() override {
|
2723
|
-
if (refs_.Unref()) delete this;
|
2724
|
-
}
|
2725
|
-
RefCountedPtr<WrappingCallSpine> Ref() {
|
2726
|
-
IncrementRefCount();
|
2727
|
-
return RefCountedPtr<WrappingCallSpine>(this);
|
2728
|
-
}
|
2729
|
-
|
2730
|
-
ClientMetadata& UnprocessedClientInitialMetadata() override {
|
2731
|
-
Crash("not for v2");
|
2732
|
-
}
|
2733
|
-
|
2734
|
-
void V2HackToStartCallWithoutACallFilterStack() override {}
|
2735
|
-
|
2736
|
-
private:
|
2737
|
-
RefCount refs_;
|
2738
|
-
ClientPromiseBasedCall* const call_;
|
2739
|
-
std::atomic<bool> sent_trailing_metadata_{false};
|
2740
|
-
Pipe<ClientMetadataHandle> client_initial_metadata_{call_->arena()};
|
2741
|
-
Pipe<ServerMetadataHandle> server_trailing_metadata_{call_->arena()};
|
2742
|
-
Latch<ServerMetadataHandle> cancel_error_;
|
2743
|
-
Latch<bool> was_cancelled_latch_;
|
2744
|
-
};
|
2745
|
-
CHECK(call_args.server_initial_metadata ==
|
2746
|
-
&server_initial_metadata_.sender);
|
2747
|
-
CHECK(call_args.client_to_server_messages ==
|
2748
|
-
&client_to_server_messages_.receiver);
|
2749
|
-
CHECK(call_args.server_to_client_messages ==
|
2750
|
-
&server_to_client_messages_.sender);
|
2751
|
-
call_args.client_initial_metadata_outstanding.Complete(true);
|
2752
|
-
return MakeRefCounted<WrappingCallSpine>(
|
2753
|
-
this, std::move(call_args.client_initial_metadata));
|
2754
|
-
}
|
2755
|
-
|
2756
|
-
private:
|
2757
|
-
// Finish the call with the given status/trailing metadata.
|
2758
|
-
void Finish(ServerMetadataHandle trailing_metadata);
|
2759
|
-
// Validate that a set of ops is valid for a client call.
|
2760
|
-
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
2761
|
-
// Commit a valid batch of operations to be executed.
|
2762
|
-
void CommitBatch(const grpc_op* ops, size_t nops,
|
2763
|
-
const Completion& completion);
|
2764
|
-
// Start the underlying promise.
|
2765
|
-
void StartPromise(ClientMetadataHandle client_initial_metadata,
|
2766
|
-
const Completion& completion, Party::BulkSpawner& spawner);
|
2767
|
-
// Start receiving initial metadata
|
2768
|
-
void StartRecvInitialMetadata(grpc_metadata_array* array,
|
2769
|
-
const Completion& completion,
|
2770
|
-
Party::BulkSpawner& spawner);
|
2771
|
-
void StartRecvStatusOnClient(
|
2772
|
-
const Completion& completion,
|
2773
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2774
|
-
Party::BulkSpawner& spawner);
|
2775
|
-
// Publish status out to the application.
|
2776
|
-
void PublishStatus(
|
2777
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2778
|
-
ServerMetadataHandle trailing_metadata);
|
2779
|
-
// Publish server initial metadata out to the application.
|
2780
|
-
void PublishInitialMetadata(ServerMetadata* metadata);
|
2781
|
-
|
2782
|
-
ClientMetadataHandle send_initial_metadata_;
|
2783
|
-
Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
|
2784
|
-
Latch<ServerMetadataHandle> server_trailing_metadata_;
|
2785
|
-
Latch<ServerMetadataHandle> cancel_error_;
|
2786
|
-
Latch<grpc_polling_entity> polling_entity_;
|
2787
|
-
Pipe<MessageHandle> client_to_server_messages_{arena()};
|
2788
|
-
Pipe<MessageHandle> server_to_client_messages_{arena()};
|
2789
|
-
bool is_trailers_only_ = false;
|
2790
|
-
bool scheduled_receive_status_ = false;
|
2791
|
-
bool scheduled_send_close_ = false;
|
2792
|
-
// True once the promise for the call is started.
|
2793
|
-
// This corresponds to sending initial metadata, or cancelling before doing
|
2794
|
-
// so.
|
2795
|
-
// In the latter case real world code sometimes does not sent the initial
|
2796
|
-
// metadata, and so gating based upon that does not work out.
|
2797
|
-
std::atomic<bool> started_{false};
|
2798
|
-
// True after the first CancelWithError call - prevents spamming cancels from
|
2799
|
-
// overflowing the party.
|
2800
|
-
std::atomic<bool> cancel_with_error_called_{false};
|
2801
|
-
// TODO(ctiller): delete when we remove the filter based API (may require some
|
2802
|
-
// cleanup in wrapped languages: they depend on this to hold slice refs)
|
2803
|
-
ServerMetadataHandle recv_initial_metadata_;
|
2804
|
-
ServerMetadataHandle recv_trailing_metadata_;
|
2805
|
-
};
|
2806
|
-
|
2807
|
-
void ClientPromiseBasedCall::StartPromise(
|
2808
|
-
ClientMetadataHandle client_initial_metadata, const Completion& completion,
|
2809
|
-
Party::BulkSpawner& spawner) {
|
2810
|
-
auto token = ClientInitialMetadataOutstandingToken::New(arena());
|
2811
|
-
spawner.Spawn(
|
2812
|
-
"call_send_initial_metadata", token.Wait(),
|
2813
|
-
[this,
|
2814
|
-
completion = AddOpToCompletion(
|
2815
|
-
completion, PendingOp::kSendInitialMetadata)](bool result) mutable {
|
2816
|
-
if (!result) FailCompletion(completion);
|
2817
|
-
FinishOpOnCompletion(&completion, PendingOp::kSendInitialMetadata);
|
2818
|
-
});
|
2819
|
-
spawner.Spawn(
|
2820
|
-
"client_promise",
|
2821
|
-
[this, client_initial_metadata = std::move(client_initial_metadata),
|
2822
|
-
token = std::move(token)]() mutable {
|
2823
|
-
return Race(
|
2824
|
-
cancel_error_.Wait(),
|
2825
|
-
Map(channel()->channel_stack()->MakeClientCallPromise(CallArgs{
|
2826
|
-
std::move(client_initial_metadata), std::move(token),
|
2827
|
-
&polling_entity_, &server_initial_metadata_.sender,
|
2828
|
-
&client_to_server_messages_.receiver,
|
2829
|
-
&server_to_client_messages_.sender}),
|
2830
|
-
[this](ServerMetadataHandle trailing_metadata) {
|
2831
|
-
// If we're cancelled the transport doesn't get to return
|
2832
|
-
// stats.
|
2833
|
-
AcceptTransportStatsFromContext();
|
2834
|
-
return trailing_metadata;
|
2835
|
-
}));
|
2836
|
-
},
|
2837
|
-
[this](ServerMetadataHandle trailing_metadata) {
|
2838
|
-
Finish(std::move(trailing_metadata));
|
2839
|
-
});
|
2840
|
-
}
|
2841
|
-
|
2842
|
-
grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
2843
|
-
size_t nops) const {
|
2844
|
-
BitSet<8> got_ops;
|
2845
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2846
|
-
const grpc_op& op = ops[op_idx];
|
2847
|
-
switch (op.op) {
|
2848
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
2849
|
-
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
2850
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2851
|
-
}
|
2852
|
-
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
2853
|
-
op.data.send_initial_metadata.metadata)) {
|
2854
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
2855
|
-
}
|
2856
|
-
break;
|
2857
|
-
case GRPC_OP_SEND_MESSAGE:
|
2858
|
-
if (!AreWriteFlagsValid(op.flags)) {
|
2859
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2860
|
-
}
|
2861
|
-
break;
|
2862
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
2863
|
-
case GRPC_OP_RECV_MESSAGE:
|
2864
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2865
|
-
break;
|
2866
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2867
|
-
if (scheduled_send_close_) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2868
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2869
|
-
break;
|
2870
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
2871
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2872
|
-
if (scheduled_receive_status_) {
|
2873
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2874
|
-
}
|
2875
|
-
break;
|
2876
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2877
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2878
|
-
return GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
2879
|
-
}
|
2880
|
-
if (got_ops.is_set(op.op)) {
|
2881
|
-
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2882
|
-
}
|
2883
|
-
got_ops.set(op.op);
|
2884
|
-
}
|
2885
|
-
return GRPC_CALL_OK;
|
2886
|
-
}
|
2887
|
-
|
2888
|
-
void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
2889
|
-
const Completion& completion) {
|
2890
|
-
Party::BulkSpawner spawner(this);
|
2891
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2892
|
-
const grpc_op& op = ops[op_idx];
|
2893
|
-
switch (op.op) {
|
2894
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
2895
|
-
if (started_.exchange(true, std::memory_order_relaxed)) break;
|
2896
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
2897
|
-
op.data.send_initial_metadata.count,
|
2898
|
-
send_initial_metadata_.get());
|
2899
|
-
PrepareOutgoingInitialMetadata(op, *send_initial_metadata_);
|
2900
|
-
if (send_deadline() != Timestamp::InfFuture()) {
|
2901
|
-
send_initial_metadata_->Set(GrpcTimeoutMetadata(), send_deadline());
|
2902
|
-
}
|
2903
|
-
send_initial_metadata_->Set(
|
2904
|
-
WaitForReady(),
|
2905
|
-
WaitForReady::ValueType{
|
2906
|
-
(op.flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
2907
|
-
(op.flags &
|
2908
|
-
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
2909
|
-
StartPromise(std::move(send_initial_metadata_), completion, spawner);
|
2910
|
-
} break;
|
2911
|
-
case GRPC_OP_RECV_INITIAL_METADATA: {
|
2912
|
-
StartRecvInitialMetadata(
|
2913
|
-
op.data.recv_initial_metadata.recv_initial_metadata, completion,
|
2914
|
-
spawner);
|
2915
|
-
} break;
|
2916
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
2917
|
-
scheduled_receive_status_ = true;
|
2918
|
-
StartRecvStatusOnClient(completion, op.data.recv_status_on_client,
|
2919
|
-
spawner);
|
2920
|
-
} break;
|
2921
|
-
case GRPC_OP_SEND_MESSAGE:
|
2922
|
-
StartSendMessage(op, completion, &client_to_server_messages_.sender,
|
2923
|
-
spawner);
|
2924
|
-
break;
|
2925
|
-
case GRPC_OP_RECV_MESSAGE:
|
2926
|
-
StartRecvMessage(
|
2927
|
-
op, completion,
|
2928
|
-
[this]() {
|
2929
|
-
return Race(server_initial_metadata_.receiver.AwaitClosed(),
|
2930
|
-
server_to_client_messages_.receiver.AwaitClosed());
|
2931
|
-
},
|
2932
|
-
&server_to_client_messages_.receiver, false, spawner);
|
2933
|
-
break;
|
2934
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2935
|
-
scheduled_send_close_ = true;
|
2936
|
-
spawner.Spawn(
|
2937
|
-
"send_close_from_client",
|
2938
|
-
[this]() {
|
2939
|
-
client_to_server_messages_.sender.Close();
|
2940
|
-
return Empty{};
|
2941
|
-
},
|
2942
|
-
[this,
|
2943
|
-
completion = AddOpToCompletion(
|
2944
|
-
completion, PendingOp::kSendCloseFromClient)](Empty) mutable {
|
2945
|
-
FinishOpOnCompletion(&completion,
|
2946
|
-
PendingOp::kSendCloseFromClient);
|
2947
|
-
});
|
2948
|
-
break;
|
2949
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2950
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2951
|
-
abort(); // unreachable
|
2952
|
-
}
|
2953
|
-
}
|
2954
|
-
}
|
2955
|
-
|
2956
|
-
grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
2957
|
-
size_t nops,
|
2958
|
-
void* notify_tag,
|
2959
|
-
bool is_notify_tag_closure) {
|
2960
|
-
if (nops == 0) {
|
2961
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
2962
|
-
return GRPC_CALL_OK;
|
2963
|
-
}
|
2964
|
-
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
2965
|
-
if (validation_result != GRPC_CALL_OK) {
|
2966
|
-
return validation_result;
|
2967
|
-
}
|
2968
|
-
Completion completion =
|
2969
|
-
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
2970
|
-
CommitBatch(ops, nops, completion);
|
2971
|
-
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
2972
|
-
return GRPC_CALL_OK;
|
2973
|
-
}
|
2974
|
-
|
2975
|
-
void ClientPromiseBasedCall::StartRecvInitialMetadata(
|
2976
|
-
grpc_metadata_array* array, const Completion& completion,
|
2977
|
-
Party::BulkSpawner& spawner) {
|
2978
|
-
spawner.Spawn(
|
2979
|
-
"recv_initial_metadata",
|
2980
|
-
[this]() {
|
2981
|
-
return Race(server_initial_metadata_.receiver.Next(),
|
2982
|
-
Map(finished(), [](Empty) {
|
2983
|
-
return NextResult<ServerMetadataHandle>(true);
|
2984
|
-
}));
|
2985
|
-
},
|
2986
|
-
[this, array,
|
2987
|
-
completion =
|
2988
|
-
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata)](
|
2989
|
-
NextResult<ServerMetadataHandle> next_metadata) mutable {
|
2990
|
-
server_initial_metadata_.sender.Close();
|
2991
|
-
ServerMetadataHandle metadata;
|
2992
|
-
if (grpc_call_trace.enabled()) {
|
2993
|
-
gpr_log(GPR_INFO, "%s[call] RecvTrailingMetadata: %s",
|
2994
|
-
DebugTag().c_str(),
|
2995
|
-
next_metadata.has_value()
|
2996
|
-
? next_metadata.value()->DebugString().c_str()
|
2997
|
-
: "null");
|
2998
|
-
}
|
2999
|
-
if (next_metadata.has_value()) {
|
3000
|
-
metadata = std::move(next_metadata.value());
|
3001
|
-
is_trailers_only_ = metadata->get(GrpcTrailersOnly()).value_or(false);
|
3002
|
-
} else {
|
3003
|
-
is_trailers_only_ = true;
|
3004
|
-
metadata = arena()->MakePooled<ServerMetadata>();
|
3005
|
-
}
|
3006
|
-
ProcessIncomingInitialMetadata(*metadata);
|
3007
|
-
PublishMetadataArray(metadata.get(), array, true);
|
3008
|
-
recv_initial_metadata_ = std::move(metadata);
|
3009
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveInitialMetadata);
|
3010
|
-
});
|
3011
|
-
}
|
3012
|
-
|
3013
|
-
void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
3014
|
-
if (grpc_call_trace.enabled()) {
|
3015
|
-
gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
|
3016
|
-
trailing_metadata->DebugString().c_str());
|
3017
|
-
}
|
3018
|
-
ResetDeadline();
|
3019
|
-
set_completed();
|
3020
|
-
client_to_server_messages_.sender.CloseWithError();
|
3021
|
-
client_to_server_messages_.receiver.CloseWithError();
|
3022
|
-
if (trailing_metadata->get(GrpcCallWasCancelled()).value_or(false)) {
|
3023
|
-
server_to_client_messages_.receiver.CloseWithError();
|
3024
|
-
server_initial_metadata_.receiver.CloseWithError();
|
3025
|
-
}
|
3026
|
-
if (auto* channelz_channel = channel()->channelz_node()) {
|
3027
|
-
if (trailing_metadata->get(GrpcStatusMetadata())
|
3028
|
-
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3029
|
-
channelz_channel->RecordCallSucceeded();
|
3030
|
-
} else {
|
3031
|
-
channelz_channel->RecordCallFailed();
|
3032
|
-
}
|
3033
|
-
}
|
3034
|
-
server_trailing_metadata_.Set(std::move(trailing_metadata));
|
3035
|
-
}
|
3036
|
-
|
3037
|
-
namespace {
|
3038
|
-
std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
|
3039
|
-
std::string out = absl::StrCat(
|
3040
|
-
trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
|
3041
|
-
? "Error received from peer"
|
3042
|
-
: "Error generated by client",
|
3043
|
-
"grpc_status: ",
|
3044
|
-
grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
|
3045
|
-
.value_or(GRPC_STATUS_UNKNOWN)));
|
3046
|
-
if (const Slice* message =
|
3047
|
-
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3048
|
-
absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
|
3049
|
-
}
|
3050
|
-
if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
|
3051
|
-
absl::StrAppend(&out, "\nStatus Context:");
|
3052
|
-
for (const std::string& annotation : *annotations) {
|
3053
|
-
absl::StrAppend(&out, "\n ", annotation);
|
3054
|
-
}
|
3055
|
-
}
|
3056
|
-
return out;
|
3057
|
-
}
|
3058
|
-
} // namespace
|
3059
|
-
|
3060
|
-
void ClientPromiseBasedCall::StartRecvStatusOnClient(
|
3061
|
-
const Completion& completion,
|
3062
|
-
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
3063
|
-
Party::BulkSpawner& spawner) {
|
3064
|
-
ForceCompletionSuccess(completion);
|
3065
|
-
spawner.Spawn(
|
3066
|
-
"recv_status_on_client", server_trailing_metadata_.Wait(),
|
3067
|
-
[this, op_args,
|
3068
|
-
completion =
|
3069
|
-
AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient)](
|
3070
|
-
ServerMetadataHandle trailing_metadata) mutable {
|
3071
|
-
const grpc_status_code status =
|
3072
|
-
trailing_metadata->get(GrpcStatusMetadata())
|
3073
|
-
.value_or(GRPC_STATUS_UNKNOWN);
|
3074
|
-
*op_args.status = status;
|
3075
|
-
Slice message_slice;
|
3076
|
-
if (Slice* message =
|
3077
|
-
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3078
|
-
message_slice = message->Ref();
|
3079
|
-
}
|
3080
|
-
SetFinalizationStatus(status, message_slice.Ref());
|
3081
|
-
*op_args.status_details = message_slice.TakeCSlice();
|
3082
|
-
if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
|
3083
|
-
*op_args.error_string =
|
3084
|
-
gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
|
3085
|
-
}
|
3086
|
-
PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata,
|
3087
|
-
true);
|
3088
|
-
recv_trailing_metadata_ = std::move(trailing_metadata);
|
3089
|
-
FinishOpOnCompletion(&completion, PendingOp::kReceiveStatusOnClient);
|
3090
|
-
});
|
3091
|
-
}
|
3092
|
-
#endif
|
3093
|
-
|
3094
|
-
///////////////////////////////////////////////////////////////////////////////
|
3095
|
-
// ServerPromiseBasedCall
|
3096
|
-
|
3097
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3098
|
-
|
3099
|
-
class ServerPromiseBasedCall final : public PromiseBasedCall,
|
3100
|
-
public ServerCallContext {
|
3101
|
-
public:
|
3102
|
-
ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
|
3103
|
-
|
3104
|
-
void OrphanCall() override {}
|
3105
|
-
void CancelWithError(grpc_error_handle) override;
|
3106
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3107
|
-
bool is_notify_tag_closure) override;
|
3108
|
-
bool is_trailers_only() const override {
|
3109
|
-
Crash("is_trailers_only not implemented for server calls");
|
3110
|
-
}
|
3111
|
-
absl::string_view GetServerAuthority() const override {
|
3112
|
-
const Slice* authority_metadata =
|
3113
|
-
client_initial_metadata_->get_pointer(HttpAuthorityMetadata());
|
3114
|
-
if (authority_metadata == nullptr) return "";
|
3115
|
-
return authority_metadata->as_string_view();
|
3116
|
-
}
|
3117
|
-
|
3118
|
-
// Polling order for the server promise stack:
|
3119
|
-
//
|
3120
|
-
// │ ┌───────────────────────────────────────┐
|
3121
|
-
// │ │ ServerPromiseBasedCall ├──► Lifetime management
|
3122
|
-
// │ ├───────────────────────────────────────┤
|
3123
|
-
// │ │ ConnectedChannel ├─┐
|
3124
|
-
// │ ├───────────────────────────────────────┤ └► Interactions with the
|
3125
|
-
// │ │ ... closest to transport filter │ transport - send/recv msgs
|
3126
|
-
// │ ├───────────────────────────────────────┤ and metadata, call phase
|
3127
|
-
// │ │ ... │ ordering
|
3128
|
-
// │ ├───────────────────────────────────────┤
|
3129
|
-
// │ │ ... closest to app filter │ ┌► Request matching, initial
|
3130
|
-
// │ ├───────────────────────────────────────┤ │ setup, publishing call to
|
3131
|
-
// │ │ Server::ChannelData::MakeCallPromise ├─┘ application
|
3132
|
-
// │ ├───────────────────────────────────────┤
|
3133
|
-
// │ │ MakeTopOfServerCallPromise ├──► Send trailing metadata
|
3134
|
-
// ▼ └───────────────────────────────────────┘
|
3135
|
-
// Polling &
|
3136
|
-
// instantiation
|
3137
|
-
// order
|
3138
|
-
|
3139
|
-
std::string DebugTag() const override {
|
3140
|
-
return absl::StrFormat("SERVER_CALL[%p]: ", this);
|
3141
|
-
}
|
3142
|
-
|
3143
|
-
ServerCallContext* server_call_context() override { return this; }
|
3144
|
-
|
3145
|
-
const void* server_stream_data() override { return server_transport_data_; }
|
3146
|
-
void PublishInitialMetadata(
|
3147
|
-
ClientMetadataHandle metadata,
|
3148
|
-
grpc_metadata_array* publish_initial_metadata) override;
|
3149
|
-
ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
|
3150
|
-
CallArgs call_args, grpc_completion_queue* cq,
|
3151
|
-
absl::FunctionRef<void(grpc_call* call)> publish) override;
|
3152
|
-
|
3153
|
-
private:
|
3154
|
-
class RecvCloseOpCancelState {
|
3155
|
-
public:
|
3156
|
-
// Request that receiver be filled in per
|
3157
|
-
// grpc_op_recv_close_on_server. Returns true if the request can
|
3158
|
-
// be fulfilled immediately. Returns false if the request will be
|
3159
|
-
// fulfilled later.
|
3160
|
-
bool ReceiveCloseOnServerOpStarted(int* receiver) {
|
3161
|
-
uintptr_t state = state_.load(std::memory_order_acquire);
|
3162
|
-
uintptr_t new_state;
|
3163
|
-
do {
|
3164
|
-
switch (state) {
|
3165
|
-
case kUnset:
|
3166
|
-
new_state = reinterpret_cast<uintptr_t>(receiver);
|
3167
|
-
break;
|
3168
|
-
case kFinishedWithFailure:
|
3169
|
-
*receiver = 1;
|
3170
|
-
return true;
|
3171
|
-
case kFinishedWithSuccess:
|
3172
|
-
*receiver = 0;
|
3173
|
-
return true;
|
3174
|
-
default:
|
3175
|
-
Crash("Two threads offered ReceiveCloseOnServerOpStarted");
|
3176
|
-
}
|
3177
|
-
} while (!state_.compare_exchange_weak(state, new_state,
|
3178
|
-
std::memory_order_acq_rel,
|
3179
|
-
std::memory_order_acquire));
|
3180
|
-
return false;
|
3181
|
-
}
|
3182
|
-
|
3183
|
-
// Mark the call as having completed.
|
3184
|
-
// Returns true if this finishes a previous
|
3185
|
-
// RequestReceiveCloseOnServer.
|
3186
|
-
bool CompleteCallWithCancelledSetTo(bool cancelled) {
|
3187
|
-
uintptr_t state = state_.load(std::memory_order_acquire);
|
3188
|
-
uintptr_t new_state;
|
3189
|
-
bool r;
|
3190
|
-
do {
|
3191
|
-
switch (state) {
|
3192
|
-
case kUnset:
|
3193
|
-
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3194
|
-
r = false;
|
3195
|
-
break;
|
3196
|
-
case kFinishedWithFailure:
|
3197
|
-
return false;
|
3198
|
-
case kFinishedWithSuccess:
|
3199
|
-
Crash("unreachable");
|
3200
|
-
default:
|
3201
|
-
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3202
|
-
r = true;
|
3203
|
-
}
|
3204
|
-
} while (!state_.compare_exchange_weak(state, new_state,
|
3205
|
-
std::memory_order_acq_rel,
|
3206
|
-
std::memory_order_acquire));
|
3207
|
-
if (r) *reinterpret_cast<int*>(state) = cancelled ? 1 : 0;
|
3208
|
-
return r;
|
3209
|
-
}
|
3210
|
-
|
3211
|
-
std::string ToString() const {
|
3212
|
-
auto state = state_.load(std::memory_order_relaxed);
|
3213
|
-
switch (state) {
|
3214
|
-
case kUnset:
|
3215
|
-
return "Unset";
|
3216
|
-
case kFinishedWithFailure:
|
3217
|
-
return "FinishedWithFailure";
|
3218
|
-
case kFinishedWithSuccess:
|
3219
|
-
return "FinishedWithSuccess";
|
3220
|
-
default:
|
3221
|
-
return absl::StrFormat("WaitingForReceiver(%p)",
|
3222
|
-
reinterpret_cast<void*>(state));
|
3223
|
-
}
|
3224
|
-
}
|
3225
|
-
|
3226
|
-
private:
|
3227
|
-
static constexpr uintptr_t kUnset = 0;
|
3228
|
-
static constexpr uintptr_t kFinishedWithFailure = 1;
|
3229
|
-
static constexpr uintptr_t kFinishedWithSuccess = 2;
|
3230
|
-
// Holds one of kUnset, kFinishedWithFailure, or
|
3231
|
-
// kFinishedWithSuccess OR an int* that wants to receive the
|
3232
|
-
// final status.
|
3233
|
-
std::atomic<uintptr_t> state_{kUnset};
|
3234
|
-
};
|
3235
|
-
|
3236
|
-
void CommitBatch(const grpc_op* ops, size_t nops,
|
3237
|
-
const Completion& completion);
|
3238
|
-
void Finish(ServerMetadataHandle result);
|
3239
|
-
|
3240
|
-
ServerInterface* const server_;
|
3241
|
-
const void* const server_transport_data_;
|
3242
|
-
PipeSender<ServerMetadataHandle>* server_initial_metadata_ = nullptr;
|
3243
|
-
PipeSender<MessageHandle>* server_to_client_messages_ = nullptr;
|
3244
|
-
PipeReceiver<MessageHandle>* client_to_server_messages_ = nullptr;
|
3245
|
-
Latch<ServerMetadataHandle> send_trailing_metadata_;
|
3246
|
-
RecvCloseOpCancelState recv_close_op_cancel_state_;
|
3247
|
-
ClientMetadataHandle client_initial_metadata_;
|
3248
|
-
Completion recv_close_completion_;
|
3249
|
-
std::atomic<bool> cancelled_{false};
|
3250
|
-
};
|
3251
|
-
|
3252
|
-
ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
3253
|
-
grpc_call_create_args* args)
|
3254
|
-
: PromiseBasedCall(arena, 0, *args),
|
3255
|
-
server_(args->server),
|
3256
|
-
server_transport_data_(args->server_transport_data) {
|
3257
|
-
global_stats().IncrementServerCallsCreated();
|
3258
|
-
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3259
|
-
if (channelz_node != nullptr) {
|
3260
|
-
channelz_node->RecordCallStarted();
|
3261
|
-
}
|
3262
|
-
ScopedContext activity_context(this);
|
3263
|
-
// TODO(yashykt): In the future, we want to also enable stats and trace
|
3264
|
-
// collecting from when the call is created at the transport. The idea is that
|
3265
|
-
// the transport would create the call tracer and pass it in as part of the
|
3266
|
-
// metadata.
|
3267
|
-
// TODO(yijiem): OpenCensus and internal Census is still using this way to
|
3268
|
-
// set server call tracer. We need to refactor them to stats plugins
|
3269
|
-
// (including removing the client channel filters).
|
3270
|
-
if (args->server != nullptr &&
|
3271
|
-
args->server->server_call_tracer_factory() != nullptr) {
|
3272
|
-
auto* server_call_tracer =
|
3273
|
-
args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
|
3274
|
-
arena, args->server->channel_args());
|
3275
|
-
if (server_call_tracer != nullptr) {
|
3276
|
-
// Note that we are setting both
|
3277
|
-
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
3278
|
-
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
3279
|
-
// promise-based world, we would just a single tracer object for each
|
3280
|
-
// stack (call, subchannel_call, server_call.)
|
3281
|
-
ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
3282
|
-
server_call_tracer, nullptr);
|
3283
|
-
ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
3284
|
-
}
|
3285
|
-
}
|
3286
|
-
args->channel->channel_stack()->stats_plugin_group->AddServerCallTracers(
|
3287
|
-
context());
|
3288
|
-
Spawn("server_promise",
|
3289
|
-
channel()->channel_stack()->MakeServerCallPromise(
|
3290
|
-
CallArgs{nullptr, ClientInitialMetadataOutstandingToken::Empty(),
|
3291
|
-
nullptr, nullptr, nullptr, nullptr}),
|
3292
|
-
[this](ServerMetadataHandle result) { Finish(std::move(result)); });
|
3293
|
-
}
|
3294
|
-
|
3295
|
-
void ServerPromiseBasedCall::Finish(ServerMetadataHandle result) {
|
3296
|
-
if (grpc_call_trace.enabled()) {
|
3297
|
-
gpr_log(GPR_INFO, "%s[call] Finish: recv_close_state:%s result:%s",
|
3298
|
-
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3299
|
-
result->DebugString().c_str());
|
3300
|
-
}
|
3301
|
-
const auto status =
|
3302
|
-
result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN);
|
3303
|
-
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3304
|
-
if (channelz_node != nullptr) {
|
3305
|
-
if (status == GRPC_STATUS_OK) {
|
3306
|
-
channelz_node->RecordCallSucceeded();
|
3307
|
-
} else {
|
3308
|
-
channelz_node->RecordCallFailed();
|
3309
|
-
}
|
3310
|
-
}
|
3311
|
-
bool was_cancelled = result->get(GrpcCallWasCancelled()).value_or(true);
|
3312
|
-
if (recv_close_op_cancel_state_.CompleteCallWithCancelledSetTo(
|
3313
|
-
was_cancelled)) {
|
3314
|
-
FinishOpOnCompletion(&recv_close_completion_,
|
3315
|
-
PendingOp::kReceiveCloseOnServer);
|
3316
|
-
}
|
3317
|
-
if (was_cancelled) set_failed_before_recv_message();
|
3318
|
-
if (server_initial_metadata_ != nullptr) {
|
3319
|
-
server_initial_metadata_->Close();
|
3320
|
-
}
|
3321
|
-
Slice message_slice;
|
3322
|
-
if (Slice* message = result->get_pointer(GrpcMessageMetadata())) {
|
3323
|
-
message_slice = message->Ref();
|
3324
|
-
}
|
3325
|
-
AcceptTransportStatsFromContext();
|
3326
|
-
SetFinalizationStatus(status, std::move(message_slice));
|
3327
|
-
set_completed();
|
3328
|
-
ResetDeadline();
|
3329
|
-
PropagateCancellationToChildren();
|
3330
|
-
}
|
3331
|
-
|
3332
|
-
grpc_call_error ValidateServerBatch(const grpc_op* ops, size_t nops) {
|
3333
|
-
BitSet<8> got_ops;
|
3334
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3335
|
-
const grpc_op& op = ops[op_idx];
|
3336
|
-
switch (op.op) {
|
3337
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
3338
|
-
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
3339
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3340
|
-
}
|
3341
|
-
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
3342
|
-
op.data.send_initial_metadata.metadata)) {
|
3343
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3344
|
-
}
|
3345
|
-
break;
|
3346
|
-
case GRPC_OP_SEND_MESSAGE:
|
3347
|
-
if (!AreWriteFlagsValid(op.flags)) {
|
3348
|
-
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3349
|
-
}
|
3350
|
-
break;
|
3351
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3352
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3353
|
-
if (!ValidateMetadata(
|
3354
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3355
|
-
op.data.send_status_from_server.trailing_metadata)) {
|
3356
|
-
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3357
|
-
}
|
3358
|
-
break;
|
3359
|
-
case GRPC_OP_RECV_MESSAGE:
|
3360
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3361
|
-
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3362
|
-
break;
|
3363
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3364
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3365
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3366
|
-
return GRPC_CALL_ERROR_NOT_ON_SERVER;
|
3367
|
-
}
|
3368
|
-
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
3369
|
-
got_ops.set(op.op);
|
3370
|
-
}
|
3371
|
-
return GRPC_CALL_OK;
|
3372
|
-
}
|
3373
|
-
|
3374
|
-
void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
3375
|
-
const Completion& completion) {
|
3376
|
-
Party::BulkSpawner spawner(this);
|
3377
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3378
|
-
const grpc_op& op = ops[op_idx];
|
3379
|
-
switch (op.op) {
|
3380
|
-
case GRPC_OP_SEND_INITIAL_METADATA: {
|
3381
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3382
|
-
PrepareOutgoingInitialMetadata(op, *metadata);
|
3383
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
3384
|
-
op.data.send_initial_metadata.count, metadata.get());
|
3385
|
-
if (grpc_call_trace.enabled()) {
|
3386
|
-
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3387
|
-
DebugTag().c_str());
|
3388
|
-
}
|
3389
|
-
QueueSend();
|
3390
|
-
spawner.Spawn(
|
3391
|
-
"call_send_initial_metadata",
|
3392
|
-
[this, metadata = std::move(metadata)]() mutable {
|
3393
|
-
EnactSend();
|
3394
|
-
return server_initial_metadata_->Push(std::move(metadata));
|
3395
|
-
},
|
3396
|
-
[this,
|
3397
|
-
completion = AddOpToCompletion(
|
3398
|
-
completion, PendingOp::kSendInitialMetadata)](bool r) mutable {
|
3399
|
-
if (!r) {
|
3400
|
-
set_failed_before_recv_message();
|
3401
|
-
FailCompletion(completion);
|
3402
|
-
}
|
3403
|
-
FinishOpOnCompletion(&completion,
|
3404
|
-
PendingOp::kSendInitialMetadata);
|
3405
|
-
});
|
3406
|
-
} break;
|
3407
|
-
case GRPC_OP_SEND_MESSAGE:
|
3408
|
-
StartSendMessage(op, completion, server_to_client_messages_, spawner);
|
3409
|
-
break;
|
3410
|
-
case GRPC_OP_RECV_MESSAGE:
|
3411
|
-
if (cancelled_.load(std::memory_order_relaxed)) {
|
3412
|
-
set_failed_before_recv_message();
|
3413
|
-
FailCompletion(completion);
|
3414
|
-
break;
|
3415
|
-
}
|
3416
|
-
StartRecvMessage(
|
3417
|
-
op, completion, []() { return []() { return Empty{}; }; },
|
3418
|
-
client_to_server_messages_, true, spawner);
|
3419
|
-
break;
|
3420
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
3421
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3422
|
-
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3423
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3424
|
-
metadata.get());
|
3425
|
-
metadata->Set(GrpcStatusMetadata(),
|
3426
|
-
op.data.send_status_from_server.status);
|
3427
|
-
if (auto* details = op.data.send_status_from_server.status_details) {
|
3428
|
-
// TODO(ctiller): this should not be a copy, but we have callers that
|
3429
|
-
// allocate and pass in a slice created with
|
3430
|
-
// grpc_slice_from_static_string and then delete the string after
|
3431
|
-
// passing it in, which shouldn't be a supported API.
|
3432
|
-
metadata->Set(GrpcMessageMetadata(),
|
3433
|
-
Slice(grpc_slice_copy(*details)));
|
3434
|
-
}
|
3435
|
-
spawner.Spawn(
|
3436
|
-
"call_send_status_from_server",
|
3437
|
-
[this, metadata = std::move(metadata)]() mutable {
|
3438
|
-
bool r = true;
|
3439
|
-
if (send_trailing_metadata_.is_set()) {
|
3440
|
-
r = false;
|
3441
|
-
} else {
|
3442
|
-
send_trailing_metadata_.Set(std::move(metadata));
|
3443
|
-
}
|
3444
|
-
return Map(WaitForSendingStarted(), [this, r](Empty) {
|
3445
|
-
server_initial_metadata_->Close();
|
3446
|
-
server_to_client_messages_->Close();
|
3447
|
-
return r;
|
3448
|
-
});
|
3449
|
-
},
|
3450
|
-
[this, completion = AddOpToCompletion(
|
3451
|
-
completion, PendingOp::kSendStatusFromServer)](
|
3452
|
-
bool ok) mutable {
|
3453
|
-
if (!ok) {
|
3454
|
-
set_failed_before_recv_message();
|
3455
|
-
FailCompletion(completion);
|
3456
|
-
}
|
3457
|
-
FinishOpOnCompletion(&completion,
|
3458
|
-
PendingOp::kSendStatusFromServer);
|
3459
|
-
});
|
3460
|
-
} break;
|
3461
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3462
|
-
if (grpc_call_trace.enabled()) {
|
3463
|
-
gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
|
3464
|
-
DebugTag().c_str(),
|
3465
|
-
recv_close_op_cancel_state_.ToString().c_str());
|
3466
|
-
}
|
3467
|
-
ForceCompletionSuccess(completion);
|
3468
|
-
recv_close_completion_ =
|
3469
|
-
AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
|
3470
|
-
if (recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
|
3471
|
-
op.data.recv_close_on_server.cancelled)) {
|
3472
|
-
FinishOpOnCompletion(&recv_close_completion_,
|
3473
|
-
PendingOp::kReceiveCloseOnServer);
|
3474
|
-
}
|
3475
|
-
break;
|
3476
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3477
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3478
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3479
|
-
abort(); // unreachable
|
3480
|
-
}
|
3481
|
-
}
|
3482
|
-
}
|
3483
|
-
|
3484
|
-
grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
3485
|
-
size_t nops,
|
3486
|
-
void* notify_tag,
|
3487
|
-
bool is_notify_tag_closure) {
|
3488
|
-
if (nops == 0) {
|
3489
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3490
|
-
return GRPC_CALL_OK;
|
3491
|
-
}
|
3492
|
-
const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
|
3493
|
-
if (validation_result != GRPC_CALL_OK) {
|
3494
|
-
return validation_result;
|
3495
|
-
}
|
3496
|
-
Completion completion =
|
3497
|
-
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
3498
|
-
CommitBatch(ops, nops, completion);
|
3499
|
-
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
3500
|
-
return GRPC_CALL_OK;
|
3501
|
-
}
|
3502
|
-
|
3503
|
-
void ServerPromiseBasedCall::CancelWithError(absl::Status error) {
|
3504
|
-
cancelled_.store(true, std::memory_order_relaxed);
|
3505
|
-
Spawn(
|
3506
|
-
"cancel_with_error",
|
3507
|
-
[this, error = std::move(error)]() {
|
3508
|
-
if (!send_trailing_metadata_.is_set()) {
|
3509
|
-
auto md = ServerMetadataFromStatus(error);
|
3510
|
-
md->Set(GrpcCallWasCancelled(), true);
|
3511
|
-
send_trailing_metadata_.Set(std::move(md));
|
3512
|
-
}
|
3513
|
-
if (server_to_client_messages_ != nullptr) {
|
3514
|
-
server_to_client_messages_->Close();
|
3515
|
-
}
|
3516
|
-
if (server_initial_metadata_ != nullptr) {
|
3517
|
-
server_initial_metadata_->Close();
|
3518
|
-
}
|
3519
|
-
return Empty{};
|
3520
|
-
},
|
3521
|
-
[](Empty) {});
|
3522
|
-
}
|
3523
|
-
#endif
|
3524
|
-
|
3525
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3526
|
-
void ServerPromiseBasedCall::PublishInitialMetadata(
|
3527
|
-
ClientMetadataHandle metadata,
|
3528
|
-
grpc_metadata_array* publish_initial_metadata) {
|
3529
|
-
if (grpc_call_trace.enabled()) {
|
3530
|
-
gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
|
3531
|
-
metadata->DebugString().c_str());
|
3532
|
-
}
|
3533
|
-
PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
|
3534
|
-
client_initial_metadata_ = std::move(metadata);
|
3535
|
-
}
|
3536
|
-
|
3537
|
-
ArenaPromise<ServerMetadataHandle>
|
3538
|
-
ServerPromiseBasedCall::MakeTopOfServerCallPromise(
|
3539
|
-
CallArgs call_args, grpc_completion_queue* cq,
|
3540
|
-
absl::FunctionRef<void(grpc_call* call)> publish) {
|
3541
|
-
SetCompletionQueue(cq);
|
3542
|
-
call_args.polling_entity->Set(
|
3543
|
-
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)));
|
3544
|
-
server_to_client_messages_ = call_args.server_to_client_messages;
|
3545
|
-
client_to_server_messages_ = call_args.client_to_server_messages;
|
3546
|
-
server_initial_metadata_ = call_args.server_initial_metadata;
|
3547
|
-
absl::optional<Timestamp> deadline =
|
3548
|
-
client_initial_metadata_->get(GrpcTimeoutMetadata());
|
3549
|
-
if (deadline.has_value()) {
|
3550
|
-
set_send_deadline(*deadline);
|
3551
|
-
UpdateDeadline(*deadline);
|
3552
|
-
}
|
3553
|
-
ProcessIncomingInitialMetadata(*client_initial_metadata_);
|
3554
|
-
ExternalRef();
|
3555
|
-
publish(c_ptr());
|
3556
|
-
return Seq(server_to_client_messages_->AwaitClosed(),
|
3557
|
-
send_trailing_metadata_.Wait());
|
3558
|
-
}
|
3559
|
-
|
3560
|
-
///////////////////////////////////////////////////////////////////////////////
|
3561
|
-
// CallSpine based Server Call
|
3562
|
-
|
3563
|
-
class ServerCallSpine final : public PipeBasedCallSpine,
|
3564
|
-
public ServerCallContext,
|
3565
|
-
public BasicPromiseBasedCall {
|
3566
|
-
public:
|
3567
|
-
ServerCallSpine(ClientMetadataHandle client_initial_metadata,
|
3568
|
-
ServerInterface* server, Channel* channel, Arena* arena);
|
3569
|
-
|
3570
|
-
// CallSpineInterface
|
3571
|
-
Pipe<ClientMetadataHandle>& client_initial_metadata() override {
|
3572
|
-
return client_initial_metadata_;
|
3573
|
-
}
|
3574
|
-
Pipe<ServerMetadataHandle>& server_initial_metadata() override {
|
3575
|
-
return server_initial_metadata_;
|
3576
|
-
}
|
3577
|
-
Pipe<MessageHandle>& client_to_server_messages() override {
|
3578
|
-
return client_to_server_messages_;
|
3579
|
-
}
|
3580
|
-
Pipe<MessageHandle>& server_to_client_messages() override {
|
3581
|
-
return server_to_client_messages_;
|
3582
|
-
}
|
3583
|
-
Latch<ServerMetadataHandle>& cancel_latch() override { return cancel_latch_; }
|
3584
|
-
Latch<bool>& was_cancelled_latch() override { return was_cancelled_latch_; }
|
3585
|
-
Party& party() override { return *this; }
|
3586
|
-
Arena* arena() override { return BasicPromiseBasedCall::arena(); }
|
3587
|
-
void IncrementRefCount() override { InternalRef("CallSpine"); }
|
3588
|
-
void Unref() override { InternalUnref("CallSpine"); }
|
3589
|
-
|
3590
|
-
// PromiseBasedCall
|
3591
|
-
void OrphanCall() override {
|
3592
|
-
ResetDeadline();
|
3593
|
-
CancelWithError(absl::CancelledError());
|
3594
|
-
}
|
3595
|
-
void CancelWithError(grpc_error_handle error) override {
|
3596
|
-
SpawnInfallible("CancelWithError", [this, error = std::move(error)] {
|
3597
|
-
auto status = ServerMetadataFromStatus(error);
|
3598
|
-
status->Set(GrpcCallWasCancelled(), true);
|
3599
|
-
PushServerTrailingMetadata(std::move(status));
|
3600
|
-
return Empty{};
|
3601
|
-
});
|
3602
|
-
}
|
3603
|
-
bool is_trailers_only() const override {
|
3604
|
-
Crash("is_trailers_only not implemented for server calls");
|
3605
|
-
}
|
3606
|
-
absl::string_view GetServerAuthority() const override {
|
3607
|
-
Crash("unimplemented");
|
3608
|
-
}
|
3609
|
-
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3610
|
-
bool is_notify_tag_closure) override;
|
3611
|
-
|
3612
|
-
bool Completed() final { Crash("unimplemented"); }
|
3613
|
-
bool failed_before_recv_message() const final { Crash("unimplemented"); }
|
3614
|
-
|
3615
|
-
ServerCallContext* server_call_context() override { return this; }
|
3616
|
-
const void* server_stream_data() override { Crash("unimplemented"); }
|
3617
|
-
void PublishInitialMetadata(
|
3618
|
-
ClientMetadataHandle metadata,
|
3619
|
-
grpc_metadata_array* publish_initial_metadata) override;
|
3620
|
-
ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
|
3621
|
-
CallArgs, grpc_completion_queue*,
|
3622
|
-
absl::FunctionRef<void(grpc_call* call)>) override {
|
3623
|
-
Crash("unimplemented");
|
3624
|
-
}
|
3625
|
-
|
3626
|
-
void V2HackToStartCallWithoutACallFilterStack() override {}
|
3627
|
-
|
3628
|
-
ClientMetadata& UnprocessedClientInitialMetadata() override {
|
3629
|
-
Crash("not for v2");
|
3630
|
-
}
|
3631
|
-
|
3632
|
-
bool RunParty() override {
|
3633
|
-
ScopedContext ctx(this);
|
3634
|
-
return Party::RunParty();
|
3635
|
-
}
|
3636
|
-
|
3637
|
-
private:
|
3638
|
-
void CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3639
|
-
bool is_notify_tag_closure);
|
3640
|
-
StatusFlag FinishRecvMessage(NextResult<MessageHandle> result);
|
3641
|
-
|
3642
|
-
std::string DebugTag() const override {
|
3643
|
-
return absl::StrFormat("SERVER_CALL_SPINE[%p]: ", this);
|
3644
|
-
}
|
3645
|
-
|
3646
|
-
// Initial metadata from client to server
|
3647
|
-
Pipe<ClientMetadataHandle> client_initial_metadata_;
|
3648
|
-
// Initial metadata from server to client
|
3649
|
-
Pipe<ServerMetadataHandle> server_initial_metadata_;
|
3650
|
-
// Messages travelling from the application to the transport.
|
3651
|
-
Pipe<MessageHandle> client_to_server_messages_;
|
3652
|
-
// Messages travelling from the transport to the application.
|
3653
|
-
Pipe<MessageHandle> server_to_client_messages_;
|
3654
|
-
// Latch that can be set to terminate the call
|
3655
|
-
Latch<ServerMetadataHandle> cancel_latch_;
|
3656
|
-
Latch<bool> was_cancelled_latch_;
|
3657
|
-
grpc_byte_buffer** recv_message_ = nullptr;
|
3658
|
-
ClientMetadataHandle client_initial_metadata_stored_;
|
3659
|
-
};
|
3660
|
-
|
3661
|
-
ServerCallSpine::ServerCallSpine(ClientMetadataHandle client_initial_metadata,
|
3662
|
-
ServerInterface* server, Channel* channel,
|
3663
|
-
Arena* arena)
|
3664
|
-
: BasicPromiseBasedCall(arena, 0, 1,
|
3665
|
-
[channel, server]() -> grpc_call_create_args {
|
3666
|
-
grpc_call_create_args args;
|
3667
|
-
args.channel = channel->Ref();
|
3668
|
-
args.server = server;
|
3669
|
-
args.parent = nullptr;
|
3670
|
-
args.propagation_mask = 0;
|
3671
|
-
args.cq = nullptr;
|
3672
|
-
args.pollset_set_alternative = nullptr;
|
3673
|
-
args.server_transport_data =
|
3674
|
-
&args; // Arbitrary non-null pointer
|
3675
|
-
args.send_deadline = Timestamp::InfFuture();
|
3676
|
-
return args;
|
3677
|
-
}()),
|
3678
|
-
client_initial_metadata_(arena),
|
3679
|
-
server_initial_metadata_(arena),
|
3680
|
-
client_to_server_messages_(arena),
|
3681
|
-
server_to_client_messages_(arena) {
|
3682
|
-
global_stats().IncrementServerCallsCreated();
|
3683
|
-
ScopedContext ctx(this);
|
3684
|
-
channel->channel_stack()->InitServerCallSpine(this);
|
3685
|
-
SpawnGuarded("push_client_initial_metadata",
|
3686
|
-
[this, md = std::move(client_initial_metadata)]() mutable {
|
3687
|
-
return Map(client_initial_metadata_.sender.Push(std::move(md)),
|
3688
|
-
[](bool r) { return StatusFlag(r); });
|
3689
|
-
});
|
3690
|
-
}
|
3691
|
-
|
3692
|
-
void ServerCallSpine::PublishInitialMetadata(
|
3693
|
-
ClientMetadataHandle metadata,
|
3694
|
-
grpc_metadata_array* publish_initial_metadata) {
|
3695
|
-
if (grpc_call_trace.enabled()) {
|
3696
|
-
gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
|
3697
|
-
metadata->DebugString().c_str());
|
3698
|
-
}
|
3699
|
-
PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
|
3700
|
-
client_initial_metadata_stored_ = std::move(metadata);
|
3701
|
-
}
|
3702
|
-
|
3703
|
-
grpc_call_error ServerCallSpine::StartBatch(const grpc_op* ops, size_t nops,
|
3704
|
-
void* notify_tag,
|
3705
|
-
bool is_notify_tag_closure) {
|
3706
|
-
if (nops == 0) {
|
3707
|
-
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3708
|
-
return GRPC_CALL_OK;
|
3709
|
-
}
|
3710
|
-
const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
|
3711
|
-
if (validation_result != GRPC_CALL_OK) {
|
3712
|
-
return validation_result;
|
3713
|
-
}
|
3714
|
-
CommitBatch(ops, nops, notify_tag, is_notify_tag_closure);
|
3715
|
-
return GRPC_CALL_OK;
|
3716
|
-
}
|
3717
|
-
|
3718
|
-
namespace {
|
3719
|
-
template <typename SetupFn>
|
3720
|
-
class MaybeOpImpl {
|
3721
|
-
public:
|
3722
|
-
using SetupResult = decltype(std::declval<SetupFn>()(grpc_op()));
|
3723
|
-
using PromiseFactory = promise_detail::OncePromiseFactory<void, SetupResult>;
|
3724
|
-
using Promise = typename PromiseFactory::Promise;
|
3725
|
-
struct Dismissed {};
|
3726
|
-
using State = absl::variant<Dismissed, PromiseFactory, Promise>;
|
3727
|
-
|
3728
|
-
// op_ is garbage but shouldn't be uninitialized
|
3729
|
-
MaybeOpImpl() : state_(Dismissed{}), op_(GRPC_OP_RECV_STATUS_ON_CLIENT) {}
|
3730
|
-
MaybeOpImpl(SetupResult result, grpc_op_type op)
|
3731
|
-
: state_(PromiseFactory(std::move(result))), op_(op) {}
|
3732
|
-
|
3733
|
-
MaybeOpImpl(const MaybeOpImpl&) = delete;
|
3734
|
-
MaybeOpImpl& operator=(const MaybeOpImpl&) = delete;
|
3735
|
-
MaybeOpImpl(MaybeOpImpl&& /*other*/) noexcept { Crash("not implemented"); }
|
3736
|
-
MaybeOpImpl& operator=(MaybeOpImpl&& other) noexcept {
|
3737
|
-
op_ = other.op_;
|
3738
|
-
if (absl::holds_alternative<Dismissed>(state_)) {
|
3739
|
-
state_.template emplace<Dismissed>();
|
3740
|
-
return *this;
|
3741
|
-
}
|
3742
|
-
// Can't move after first poll => Promise is not an option
|
3743
|
-
state_.template emplace<PromiseFactory>(
|
3744
|
-
std::move(absl::get<PromiseFactory>(other.state_)));
|
3745
|
-
return *this;
|
3746
|
-
}
|
3747
|
-
|
3748
|
-
Poll<StatusFlag> operator()() {
|
3749
|
-
if (absl::holds_alternative<Dismissed>(state_)) return Success{};
|
3750
|
-
if (absl::holds_alternative<PromiseFactory>(state_)) {
|
3751
|
-
auto& factory = absl::get<PromiseFactory>(state_);
|
3752
|
-
auto promise = factory.Make();
|
3753
|
-
state_.template emplace<Promise>(std::move(promise));
|
3754
|
-
}
|
3755
|
-
if (grpc_call_trace.enabled()) {
|
3756
|
-
gpr_log(GPR_INFO, "%sBeginPoll %s",
|
3757
|
-
Activity::current()->DebugTag().c_str(), OpName(op_).c_str());
|
3758
|
-
}
|
3759
|
-
auto& promise = absl::get<Promise>(state_);
|
3760
|
-
auto r = poll_cast<StatusFlag>(promise());
|
3761
|
-
if (grpc_call_trace.enabled()) {
|
3762
|
-
gpr_log(GPR_INFO, "%sEndPoll %s --> %s",
|
3763
|
-
Activity::current()->DebugTag().c_str(), OpName(op_).c_str(),
|
3764
|
-
r.pending() ? "PENDING" : (r.value().ok() ? "OK" : "FAILURE"));
|
3765
|
-
}
|
3766
|
-
return r;
|
3767
|
-
}
|
3768
|
-
|
3769
|
-
private:
|
3770
|
-
GPR_NO_UNIQUE_ADDRESS State state_;
|
3771
|
-
GPR_NO_UNIQUE_ADDRESS grpc_op_type op_;
|
3772
|
-
|
3773
|
-
static std::string OpName(grpc_op_type op) {
|
3774
|
-
switch (op) {
|
3775
|
-
case GRPC_OP_SEND_INITIAL_METADATA:
|
3776
|
-
return "SendInitialMetadata";
|
3777
|
-
case GRPC_OP_SEND_MESSAGE:
|
3778
|
-
return "SendMessage";
|
3779
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3780
|
-
return "SendStatusFromServer";
|
3781
|
-
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3782
|
-
return "SendCloseFromClient";
|
3783
|
-
case GRPC_OP_RECV_MESSAGE:
|
3784
|
-
return "RecvMessage";
|
3785
|
-
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3786
|
-
return "RecvCloseOnServer";
|
3787
|
-
case GRPC_OP_RECV_INITIAL_METADATA:
|
3788
|
-
return "RecvInitialMetadata";
|
3789
|
-
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3790
|
-
return "RecvStatusOnClient";
|
3791
|
-
}
|
3792
|
-
return absl::StrCat("UnknownOp(", op, ")");
|
3793
|
-
}
|
3794
|
-
|
3795
|
-
static State MoveState(State& state) {
|
3796
|
-
if (absl::holds_alternative<Dismissed>(state)) return Dismissed{};
|
3797
|
-
// Can't move after first poll => Promise is not an option
|
3798
|
-
return std::move(absl::get<PromiseFactory>(state));
|
3799
|
-
}
|
3800
|
-
};
|
3801
|
-
|
3802
|
-
// MaybeOp captures a fairly complicated dance we need to do for the batch API.
|
3803
|
-
// We first check if an op is included or not, and if it is, we run the setup
|
3804
|
-
// function in the context of the API call (NOT in the call party).
|
3805
|
-
// This setup function returns a promise factory which we'll then run *in* the
|
3806
|
-
// party to do initial setup, and have it return the promise that we'll
|
3807
|
-
// ultimately poll on til completion.
|
3808
|
-
// Once we express our surface API in terms of core internal types this whole
|
3809
|
-
// dance will go away.
|
3810
|
-
template <typename SetupFn>
|
3811
|
-
auto MaybeOp(const grpc_op* ops, uint8_t idx, SetupFn setup) {
|
3812
|
-
if (idx == 255) {
|
3813
|
-
return MaybeOpImpl<SetupFn>();
|
3814
|
-
} else {
|
3815
|
-
return MaybeOpImpl<SetupFn>(setup(ops[idx]), ops[idx].op);
|
3816
|
-
}
|
3817
|
-
}
|
3818
|
-
|
3819
|
-
template <typename F>
|
3820
|
-
class PollBatchLogger {
|
3821
|
-
public:
|
3822
|
-
PollBatchLogger(void* tag, F f) : tag_(tag), f_(std::move(f)) {}
|
3823
|
-
|
3824
|
-
auto operator()() {
|
3825
|
-
if (grpc_call_trace.enabled()) {
|
3826
|
-
gpr_log(GPR_INFO, "Poll batch %p", tag_);
|
3827
|
-
}
|
3828
|
-
auto r = f_();
|
3829
|
-
if (grpc_call_trace.enabled()) {
|
3830
|
-
gpr_log(GPR_INFO, "Poll batch %p --> %s", tag_, ResultString(r).c_str());
|
3831
|
-
}
|
3832
|
-
return r;
|
3833
|
-
}
|
3834
|
-
|
3835
|
-
private:
|
3836
|
-
template <typename T>
|
3837
|
-
static std::string ResultString(Poll<T> r) {
|
3838
|
-
if (r.pending()) return "PENDING";
|
3839
|
-
return ResultString(r.value());
|
3840
|
-
}
|
3841
|
-
static std::string ResultString(Empty) { return "DONE"; }
|
3842
|
-
|
3843
|
-
void* tag_;
|
3844
|
-
F f_;
|
3845
|
-
};
|
3846
|
-
|
3847
|
-
template <typename F>
|
3848
|
-
PollBatchLogger<F> LogPollBatch(void* tag, F f) {
|
3849
|
-
return PollBatchLogger<F>(tag, std::move(f));
|
3850
|
-
}
|
3851
|
-
} // namespace
|
3852
|
-
|
3853
|
-
StatusFlag ServerCallSpine::FinishRecvMessage(
|
3854
|
-
NextResult<MessageHandle> result) {
|
3855
|
-
if (result.has_value()) {
|
3856
|
-
MessageHandle& message = *result;
|
3857
|
-
NoteLastMessageFlags(message->flags());
|
3858
|
-
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
3859
|
-
(incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
3860
|
-
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
3861
|
-
nullptr, 0, incoming_compression_algorithm());
|
3862
|
-
} else {
|
3863
|
-
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
3864
|
-
}
|
3865
|
-
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
3866
|
-
&(*recv_message_)->data.raw.slice_buffer);
|
3867
|
-
if (grpc_call_trace.enabled()) {
|
3868
|
-
gpr_log(GPR_INFO,
|
3869
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3870
|
-
"finishes: received %" PRIdPTR " byte message",
|
3871
|
-
DebugTag().c_str(),
|
3872
|
-
(*recv_message_)->data.raw.slice_buffer.length);
|
3873
|
-
}
|
3874
|
-
recv_message_ = nullptr;
|
3875
|
-
return Success{};
|
3876
|
-
}
|
3877
|
-
if (result.cancelled()) {
|
3878
|
-
if (grpc_call_trace.enabled()) {
|
3879
|
-
gpr_log(GPR_INFO,
|
3880
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3881
|
-
"finishes: received end-of-stream with error",
|
3882
|
-
DebugTag().c_str());
|
3883
|
-
}
|
3884
|
-
*recv_message_ = nullptr;
|
3885
|
-
recv_message_ = nullptr;
|
3886
|
-
return Failure{};
|
3887
|
-
}
|
3888
|
-
if (grpc_call_trace.enabled()) {
|
3889
|
-
gpr_log(GPR_INFO,
|
3890
|
-
"%s[call] RecvMessage: outstanding_recv "
|
3891
|
-
"finishes: received end-of-stream",
|
3892
|
-
DebugTag().c_str());
|
3893
|
-
}
|
3894
|
-
*recv_message_ = nullptr;
|
3895
|
-
recv_message_ = nullptr;
|
3896
|
-
return Success{};
|
3897
|
-
}
|
3898
|
-
|
3899
|
-
void ServerCallSpine::CommitBatch(const grpc_op* ops, size_t nops,
|
3900
|
-
void* notify_tag,
|
3901
|
-
bool is_notify_tag_closure) {
|
3902
|
-
std::array<uint8_t, 8> got_ops{255, 255, 255, 255, 255, 255, 255, 255};
|
3903
|
-
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3904
|
-
const grpc_op& op = ops[op_idx];
|
3905
|
-
got_ops[op.op] = op_idx;
|
3906
|
-
}
|
3907
|
-
if (!is_notify_tag_closure) grpc_cq_begin_op(cq(), notify_tag);
|
3908
|
-
auto send_initial_metadata = MaybeOp(
|
3909
|
-
ops, got_ops[GRPC_OP_SEND_INITIAL_METADATA], [this](const grpc_op& op) {
|
3910
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3911
|
-
PrepareOutgoingInitialMetadata(op, *metadata);
|
3912
|
-
CToMetadata(op.data.send_initial_metadata.metadata,
|
3913
|
-
op.data.send_initial_metadata.count, metadata.get());
|
3914
|
-
if (grpc_call_trace.enabled()) {
|
3915
|
-
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3916
|
-
DebugTag().c_str());
|
3917
|
-
}
|
3918
|
-
return [this, metadata = std::move(metadata)]() mutable {
|
3919
|
-
return Map(server_initial_metadata_.sender.Push(std::move(metadata)),
|
3920
|
-
[this](bool r) {
|
3921
|
-
server_initial_metadata_.sender.Close();
|
3922
|
-
return StatusFlag(r);
|
3923
|
-
});
|
3924
|
-
};
|
3925
|
-
});
|
3926
|
-
auto send_message =
|
3927
|
-
MaybeOp(ops, got_ops[GRPC_OP_SEND_MESSAGE], [this](const grpc_op& op) {
|
3928
|
-
SliceBuffer send;
|
3929
|
-
grpc_slice_buffer_swap(
|
3930
|
-
&op.data.send_message.send_message->data.raw.slice_buffer,
|
3931
|
-
send.c_slice_buffer());
|
3932
|
-
auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
|
3933
|
-
return [this, msg = std::move(msg)]() mutable {
|
3934
|
-
return Map(server_to_client_messages_.sender.Push(std::move(msg)),
|
3935
|
-
[](bool r) { return StatusFlag(r); });
|
3936
|
-
};
|
3937
|
-
});
|
3938
|
-
auto send_trailing_metadata = MaybeOp(
|
3939
|
-
ops, got_ops[GRPC_OP_SEND_STATUS_FROM_SERVER], [this](const grpc_op& op) {
|
3940
|
-
auto metadata = arena()->MakePooled<ServerMetadata>();
|
3941
|
-
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3942
|
-
op.data.send_status_from_server.trailing_metadata_count,
|
3943
|
-
metadata.get());
|
3944
|
-
metadata->Set(GrpcStatusMetadata(),
|
3945
|
-
op.data.send_status_from_server.status);
|
3946
|
-
if (auto* details = op.data.send_status_from_server.status_details) {
|
3947
|
-
// TODO(ctiller): this should not be a copy, but we have
|
3948
|
-
// callers that allocate and pass in a slice created with
|
3949
|
-
// grpc_slice_from_static_string and then delete the string
|
3950
|
-
// after passing it in, which shouldn't be a supported API.
|
3951
|
-
metadata->Set(GrpcMessageMetadata(),
|
3952
|
-
Slice(grpc_slice_copy(*details)));
|
3953
|
-
}
|
3954
|
-
CHECK(metadata != nullptr);
|
3955
|
-
return [this, metadata = std::move(metadata)]() mutable {
|
3956
|
-
CHECK(metadata != nullptr);
|
3957
|
-
return [this,
|
3958
|
-
metadata = std::move(metadata)]() mutable -> Poll<Success> {
|
3959
|
-
CHECK(metadata != nullptr);
|
3960
|
-
PushServerTrailingMetadata(std::move(metadata));
|
3961
|
-
return Success{};
|
3962
|
-
};
|
3963
|
-
};
|
3964
|
-
});
|
3965
|
-
auto recv_message =
|
3966
|
-
MaybeOp(ops, got_ops[GRPC_OP_RECV_MESSAGE], [this](const grpc_op& op) {
|
3967
|
-
CHECK_EQ(recv_message_, nullptr);
|
3968
|
-
recv_message_ = op.data.recv_message.recv_message;
|
3969
|
-
return [this]() mutable {
|
3970
|
-
return Map(client_to_server_messages_.receiver.Next(),
|
3971
|
-
[this](NextResult<MessageHandle> msg) {
|
3972
|
-
return FinishRecvMessage(std::move(msg));
|
3973
|
-
});
|
3974
|
-
};
|
3975
|
-
});
|
3976
|
-
auto primary_ops = AllOk<StatusFlag>(
|
3977
|
-
TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
|
3978
|
-
std::move(send_message)),
|
3979
|
-
std::move(send_trailing_metadata)),
|
3980
|
-
std::move(recv_message));
|
3981
|
-
if (got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER] != 255) {
|
3982
|
-
auto recv_trailing_metadata = MaybeOp(
|
3983
|
-
ops, got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER], [this](const grpc_op& op) {
|
3984
|
-
return [this, cancelled = op.data.recv_close_on_server.cancelled]() {
|
3985
|
-
return Map(WasCancelled(),
|
3986
|
-
[cancelled, this](bool result) -> Success {
|
3987
|
-
ResetDeadline();
|
3988
|
-
*cancelled = result ? 1 : 0;
|
3989
|
-
return Success{};
|
3990
|
-
});
|
3991
|
-
};
|
3992
|
-
});
|
3993
|
-
SpawnInfallible(
|
3994
|
-
"final-batch",
|
3995
|
-
[primary_ops = std::move(primary_ops),
|
3996
|
-
recv_trailing_metadata = std::move(recv_trailing_metadata),
|
3997
|
-
is_notify_tag_closure, notify_tag, this]() mutable {
|
3998
|
-
return LogPollBatch(
|
3999
|
-
notify_tag,
|
4000
|
-
Seq(std::move(primary_ops), std::move(recv_trailing_metadata),
|
4001
|
-
[is_notify_tag_closure, notify_tag, this](StatusFlag) {
|
4002
|
-
return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
|
4003
|
-
absl::OkStatus(), cq());
|
4004
|
-
}));
|
4005
|
-
});
|
4006
|
-
} else {
|
4007
|
-
SpawnInfallible("batch", [primary_ops = std::move(primary_ops),
|
4008
|
-
is_notify_tag_closure, notify_tag,
|
4009
|
-
this]() mutable {
|
4010
|
-
return LogPollBatch(
|
4011
|
-
notify_tag,
|
4012
|
-
Seq(std::move(primary_ops),
|
4013
|
-
[is_notify_tag_closure, notify_tag, this](StatusFlag r) {
|
4014
|
-
return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
|
4015
|
-
StatusCast<grpc_error_handle>(r), cq());
|
4016
|
-
}));
|
4017
|
-
});
|
4018
|
-
}
|
4019
|
-
}
|
4020
|
-
|
4021
|
-
RefCountedPtr<CallSpineInterface> MakeServerCall(
|
4022
|
-
ClientMetadataHandle client_initial_metadata, ServerInterface* server,
|
4023
|
-
Channel* channel, Arena* arena) {
|
4024
|
-
return RefCountedPtr<ServerCallSpine>(arena->New<ServerCallSpine>(
|
4025
|
-
std::move(client_initial_metadata), server, channel, arena));
|
4026
|
-
}
|
4027
|
-
#else
|
4028
|
-
RefCountedPtr<CallSpineInterface> MakeServerCall(ClientMetadataHandle,
|
4029
|
-
ServerInterface*, Channel*,
|
4030
|
-
Arena*) {
|
4031
|
-
Crash("not implemented");
|
4032
|
-
}
|
4033
|
-
#endif
|
4034
|
-
|
4035
|
-
} // namespace grpc_core
|
4036
|
-
|
4037
|
-
///////////////////////////////////////////////////////////////////////////////
|
4038
|
-
// C-based API
|
4039
|
-
|
4040
|
-
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
4041
|
-
grpc_core::ExecCtx exec_ctx;
|
4042
|
-
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
4043
|
-
}
|
4044
|
-
|
4045
|
-
size_t grpc_call_get_initial_size_estimate() {
|
4046
|
-
return grpc_core::FilterStackCall::InitialSizeEstimate();
|
4047
|
-
}
|
4048
|
-
|
4049
|
-
grpc_error_handle grpc_call_create(grpc_call_create_args* args,
|
4050
|
-
grpc_call** out_call) {
|
4051
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
4052
|
-
if (grpc_core::IsPromiseBasedClientCallEnabled() &&
|
4053
|
-
args->server_transport_data == nullptr && args->channel->is_promising()) {
|
4054
|
-
return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
|
4055
|
-
args, out_call);
|
4056
|
-
}
|
4057
|
-
#endif
|
4058
|
-
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
4059
|
-
if (grpc_core::IsPromiseBasedServerCallEnabled() &&
|
4060
|
-
args->server_transport_data != nullptr && args->channel->is_promising()) {
|
4061
|
-
return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
|
4062
|
-
args, out_call);
|
4063
|
-
}
|
4064
|
-
#endif
|
4065
|
-
return grpc_core::FilterStackCall::Create(args, out_call);
|
4066
|
-
}
|
4067
|
-
|
4068
|
-
void grpc_call_set_completion_queue(grpc_call* call,
|
4069
|
-
grpc_completion_queue* cq) {
|
4070
|
-
grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
|
4071
|
-
}
|
4072
|
-
|
4073
|
-
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
4074
|
-
|
4075
|
-
void grpc_call_unref(grpc_call* c) {
|
4076
|
-
grpc_core::ExecCtx exec_ctx;
|
4077
|
-
grpc_core::Call::FromC(c)->ExternalUnref();
|
401
|
+
void grpc_call_unref(grpc_call* c) {
|
402
|
+
grpc_core::ExecCtx exec_ctx;
|
403
|
+
grpc_core::Call::FromC(c)->ExternalUnref();
|
4078
404
|
}
|
4079
405
|
|
4080
406
|
char* grpc_call_get_peer(grpc_call* call) {
|
4081
407
|
return grpc_core::Call::FromC(call)->GetPeer();
|
4082
408
|
}
|
4083
409
|
|
4084
|
-
grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element) {
|
4085
|
-
return grpc_core::FilterStackCall::FromTopElem(surface_element)->c_ptr();
|
4086
|
-
}
|
4087
|
-
|
4088
410
|
grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
|
4089
411
|
GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved));
|
4090
412
|
CHECK_EQ(reserved, nullptr);
|
@@ -4121,7 +443,7 @@ void grpc_call_cancel_internal(grpc_call* call) {
|
|
4121
443
|
|
4122
444
|
grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
|
4123
445
|
grpc_call* call) {
|
4124
|
-
return grpc_core::Call::FromC(call)->
|
446
|
+
return grpc_core::Call::FromC(call)->incoming_compression_algorithm();
|
4125
447
|
}
|
4126
448
|
|
4127
449
|
uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
|
@@ -4165,13 +487,17 @@ grpc_call_error grpc_call_start_batch_and_execute(grpc_call* call,
|
|
4165
487
|
return grpc_core::Call::FromC(call)->StartBatch(ops, nops, closure, true);
|
4166
488
|
}
|
4167
489
|
|
4168
|
-
void
|
4169
|
-
|
4170
|
-
|
490
|
+
void grpc_call_tracer_set(grpc_call* call,
|
491
|
+
grpc_core::ClientCallTracer* tracer) {
|
492
|
+
grpc_core::Arena* arena = grpc_call_get_arena(call);
|
493
|
+
return arena->SetContext<grpc_core::CallTracerAnnotationInterface>(tracer);
|
4171
494
|
}
|
4172
495
|
|
4173
|
-
void*
|
4174
|
-
|
496
|
+
void* grpc_call_tracer_get(grpc_call* call) {
|
497
|
+
grpc_core::Arena* arena = grpc_call_get_arena(call);
|
498
|
+
auto* call_tracer =
|
499
|
+
arena->GetContext<grpc_core::CallTracerAnnotationInterface>();
|
500
|
+
return call_tracer;
|
4175
501
|
}
|
4176
502
|
|
4177
503
|
uint8_t grpc_call_is_client(grpc_call* call) {
|