grpc 1.46.3 → 1.53.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of grpc might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Makefile +419 -241
- data/include/grpc/byte_buffer.h +76 -1
- data/include/grpc/byte_buffer_reader.h +19 -1
- data/include/grpc/compression.h +2 -2
- data/include/grpc/event_engine/endpoint_config.h +11 -5
- data/include/grpc/event_engine/event_engine.h +129 -39
- data/include/grpc/event_engine/internal/memory_allocator_impl.h +1 -1
- data/include/grpc/event_engine/internal/slice_cast.h +67 -0
- data/include/grpc/event_engine/memory_allocator.h +1 -16
- data/include/grpc/event_engine/port.h +1 -1
- data/include/grpc/event_engine/slice.h +306 -0
- data/include/grpc/event_engine/slice_buffer.h +159 -0
- data/include/grpc/fork.h +25 -1
- data/include/grpc/grpc.h +4 -14
- data/include/grpc/grpc_posix.h +1 -1
- data/include/grpc/grpc_security.h +11 -0
- data/include/grpc/impl/codegen/atm.h +3 -71
- data/include/grpc/impl/codegen/atm_gcc_atomic.h +3 -67
- data/include/grpc/impl/codegen/atm_gcc_sync.h +3 -61
- data/include/grpc/impl/codegen/atm_windows.h +3 -108
- data/include/grpc/impl/codegen/byte_buffer.h +4 -78
- data/include/grpc/impl/codegen/byte_buffer_reader.h +4 -19
- data/include/grpc/impl/codegen/compression_types.h +5 -83
- data/include/grpc/impl/codegen/connectivity_state.h +5 -21
- data/include/grpc/impl/codegen/fork.h +4 -25
- data/include/grpc/impl/codegen/gpr_types.h +4 -35
- data/include/grpc/impl/codegen/grpc_types.h +5 -791
- data/include/grpc/impl/codegen/log.h +3 -86
- data/include/grpc/impl/codegen/port_platform.h +3 -699
- data/include/grpc/impl/codegen/propagation_bits.h +3 -28
- data/include/grpc/impl/codegen/slice.h +4 -107
- data/include/grpc/impl/codegen/status.h +4 -131
- data/include/grpc/impl/codegen/sync.h +3 -42
- data/include/grpc/impl/codegen/sync_abseil.h +3 -12
- data/include/grpc/impl/codegen/sync_custom.h +3 -14
- data/include/grpc/impl/codegen/sync_generic.h +3 -25
- data/include/grpc/impl/codegen/sync_posix.h +3 -28
- data/include/grpc/impl/codegen/sync_windows.h +3 -16
- data/include/grpc/impl/compression_types.h +109 -0
- data/include/grpc/impl/connectivity_state.h +47 -0
- data/include/grpc/impl/grpc_types.h +836 -0
- data/include/grpc/impl/propagation_bits.h +54 -0
- data/include/grpc/impl/slice_type.h +112 -0
- data/include/grpc/load_reporting.h +1 -1
- data/include/grpc/module.modulemap +5 -1
- data/include/grpc/slice.h +1 -1
- data/include/grpc/status.h +131 -1
- data/include/grpc/support/atm.h +70 -1
- data/include/grpc/support/atm_gcc_atomic.h +59 -1
- data/include/grpc/support/atm_gcc_sync.h +58 -1
- data/include/grpc/support/atm_windows.h +105 -1
- data/include/grpc/support/log.h +87 -1
- data/include/grpc/support/log_windows.h +1 -1
- data/include/grpc/support/port_platform.h +767 -1
- data/include/grpc/support/string_util.h +1 -1
- data/include/grpc/support/sync.h +35 -2
- data/include/grpc/support/sync_abseil.h +11 -1
- data/include/grpc/support/sync_custom.h +13 -1
- data/include/grpc/support/sync_generic.h +24 -1
- data/include/grpc/support/sync_posix.h +27 -1
- data/include/grpc/support/sync_windows.h +15 -1
- data/include/grpc/support/time.h +31 -6
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +148 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +52 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +29 -0
- data/src/core/ext/filters/census/grpc_context.cc +19 -17
- data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +67 -61
- data/src/core/ext/filters/channel_idle/channel_idle_filter.h +26 -6
- data/src/core/ext/filters/channel_idle/idle_filter_state.h +6 -4
- data/src/core/ext/filters/client_channel/backend_metric.cc +19 -12
- data/src/core/ext/filters/client_channel/backend_metric.h +21 -11
- data/src/core/ext/filters/client_channel/backup_poller.cc +34 -34
- data/src/core/ext/filters/client_channel/backup_poller.h +26 -28
- data/src/core/ext/filters/client_channel/channel_connectivity.cc +89 -44
- data/src/core/ext/filters/client_channel/client_channel.cc +401 -451
- data/src/core/ext/filters/client_channel/client_channel.h +68 -26
- data/src/core/ext/filters/client_channel/client_channel_channelz.cc +19 -23
- data/src/core/ext/filters/client_channel/client_channel_channelz.h +33 -23
- data/src/core/ext/filters/client_channel/client_channel_factory.cc +19 -43
- data/src/core/ext/filters/client_channel/client_channel_factory.h +11 -11
- data/src/core/ext/filters/client_channel/client_channel_plugin.cc +22 -47
- data/src/core/ext/filters/client_channel/client_channel_service_config.cc +153 -0
- data/src/core/ext/filters/client_channel/{resolver_result_parsing.h → client_channel_service_config.h} +33 -27
- data/src/core/ext/filters/client_channel/config_selector.cc +1 -0
- data/src/core/ext/filters/client_channel/config_selector.h +29 -23
- data/src/core/ext/filters/client_channel/connector.h +19 -10
- data/src/core/ext/filters/client_channel/dynamic_filters.cc +40 -54
- data/src/core/ext/filters/client_channel/dynamic_filters.h +20 -11
- data/src/core/ext/filters/client_channel/global_subchannel_pool.cc +2 -0
- data/src/core/ext/filters/client_channel/global_subchannel_pool.h +24 -21
- data/src/core/ext/filters/client_channel/health/health_check_client.cc +16 -4
- data/src/core/ext/filters/client_channel/health/health_check_client.h +5 -3
- data/src/core/ext/filters/client_channel/http_proxy.cc +100 -112
- data/src/core/ext/filters/client_channel/http_proxy.h +38 -31
- data/src/core/ext/filters/client_channel/lb_call_state_internal.h +39 -0
- data/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc +8 -5
- data/src/core/ext/filters/client_channel/lb_policy/address_filtering.h +4 -3
- data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +49 -0
- data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc +37 -16
- data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h +18 -13
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +71 -131
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h +41 -24
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +410 -375
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h +26 -27
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc +15 -3
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h +9 -7
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc +18 -20
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +27 -22
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc +28 -19
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +26 -25
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc +338 -0
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h +57 -0
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric_internal.h +117 -0
- data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc +1129 -0
- data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h +94 -0
- data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +174 -180
- data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc +366 -407
- data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +491 -405
- data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h +19 -9
- data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +619 -634
- data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +261 -234
- data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +165 -143
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc +128 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.h +71 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc +972 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +223 -220
- data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +214 -153
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_attributes.cc +42 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/{xds.h → xds_attributes.h} +19 -15
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h +3 -3
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +178 -256
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +199 -210
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +382 -368
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +819 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h +67 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc +369 -0
- data/src/core/ext/filters/client_channel/local_subchannel_pool.cc +4 -0
- data/src/core/ext/filters/client_channel/local_subchannel_pool.h +22 -21
- data/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc +34 -19
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +531 -163
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +50 -44
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +31 -30
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +89 -84
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +300 -209
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +82 -61
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +18 -18
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +20 -19
- data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +2 -0
- data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h +22 -22
- data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +54 -22
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +21 -34
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +19 -5
- data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +135 -63
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +136 -68
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +29 -14
- data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +21 -27
- data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +368 -305
- data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h +6 -4
- data/src/core/ext/filters/client_channel/retry_filter.cc +166 -163
- data/src/core/ext/filters/client_channel/retry_filter.h +4 -3
- data/src/core/ext/filters/client_channel/retry_service_config.cc +197 -229
- data/src/core/ext/filters/client_channel/retry_service_config.h +29 -26
- data/src/core/ext/filters/client_channel/retry_throttle.cc +28 -37
- data/src/core/ext/filters/client_channel/retry_throttle.h +36 -29
- data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +34 -17
- data/src/core/ext/filters/client_channel/subchannel.cc +332 -384
- data/src/core/ext/filters/client_channel/subchannel.h +112 -82
- data/src/core/ext/filters/client_channel/subchannel_interface_internal.h +38 -0
- data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc +17 -77
- data/src/core/ext/filters/client_channel/subchannel_pool_interface.h +41 -47
- data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +34 -108
- data/src/core/ext/filters/client_channel/subchannel_stream_client.h +20 -12
- data/src/core/ext/filters/deadline/deadline_filter.cc +94 -76
- data/src/core/ext/filters/deadline/deadline_filter.h +14 -12
- data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +44 -22
- data/src/core/ext/filters/fault_injection/fault_injection_filter.h +20 -11
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc +118 -0
- data/src/core/ext/filters/fault_injection/{service_config_parser.h → fault_injection_service_config_parser.h} +34 -13
- data/src/core/ext/filters/http/client/http_client_filter.cc +76 -54
- data/src/core/ext/filters/http/client/http_client_filter.h +36 -23
- data/src/core/ext/filters/http/client_authority_filter.cc +33 -38
- data/src/core/ext/filters/http/client_authority_filter.h +27 -24
- data/src/core/ext/filters/http/http_filters_plugin.cc +46 -55
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +307 -0
- data/src/core/ext/filters/http/message_compress/compression_filter.h +139 -0
- data/src/core/ext/filters/http/server/http_server_filter.cc +111 -271
- data/src/core/ext/filters/http/server/http_server_filter.h +58 -24
- data/src/core/ext/filters/message_size/message_size_filter.cc +117 -137
- data/src/core/ext/filters/message_size/message_size_filter.h +34 -17
- data/src/core/ext/filters/rbac/rbac_filter.cc +29 -16
- data/src/core/ext/filters/rbac/rbac_filter.h +11 -3
- data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +736 -523
- data/src/core/ext/filters/rbac/rbac_service_config_parser.h +18 -7
- data/src/core/ext/filters/server_config_selector/server_config_selector.h +20 -8
- data/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +28 -12
- data/src/core/ext/filters/server_config_selector/server_config_selector_filter.h +4 -3
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +219 -0
- data/src/core/ext/filters/stateful_session/stateful_session_filter.h +66 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc +82 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +93 -0
- data/src/core/ext/transport/chttp2/alpn/alpn.cc +18 -18
- data/src/core/ext/transport/chttp2/alpn/alpn.h +24 -24
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +185 -241
- data/src/core/ext/transport/chttp2/client/chttp2_connector.h +38 -30
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +204 -211
- data/src/core/ext/transport/chttp2/server/chttp2_server.h +26 -26
- data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +24 -24
- data/src/core/ext/transport/chttp2/transport/bin_decoder.h +35 -34
- data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +43 -38
- data/src/core/ext/transport/chttp2/transport/bin_encoder.h +35 -31
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +801 -1067
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +28 -23
- data/src/core/ext/transport/chttp2/transport/context_list.cc +23 -19
- data/src/core/ext/transport/chttp2/transport/context_list.h +32 -29
- data/src/core/ext/transport/chttp2/transport/decode_huff.cc +251 -0
- data/src/core/ext/transport/chttp2/transport/decode_huff.h +971 -0
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +350 -300
- data/src/core/ext/transport/chttp2/transport/flow_control.h +248 -325
- data/src/core/ext/transport/chttp2/transport/frame.h +22 -26
- data/src/core/ext/transport/chttp2/transport/frame_data.cc +85 -234
- data/src/core/ext/transport/chttp2/transport/frame_data.h +38 -61
- data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +41 -36
- data/src/core/ext/transport/chttp2/transport/frame_goaway.h +24 -22
- data/src/core/ext/transport/chttp2/transport/frame_ping.cc +31 -25
- data/src/core/ext/transport/chttp2/transport/frame_ping.h +25 -22
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +35 -26
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +24 -21
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +44 -77
- data/src/core/ext/transport/chttp2/transport/frame_settings.h +27 -23
- data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +34 -33
- data/src/core/ext/transport/chttp2/transport/frame_window_update.h +24 -21
- data/src/core/ext/transport/chttp2/transport/hpack_constants.h +10 -4
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +221 -372
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +338 -158
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +6 -1
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +16 -5
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +516 -751
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +38 -27
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +86 -76
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +79 -49
- data/src/core/ext/transport/chttp2/transport/http2_settings.cc +21 -19
- data/src/core/ext/transport/chttp2/transport/http2_settings.h +39 -38
- data/src/core/ext/transport/chttp2/transport/http_trace.cc +19 -0
- data/src/core/ext/transport/chttp2/transport/http_trace.h +24 -0
- data/src/core/ext/transport/chttp2/transport/huffsyms.cc +20 -20
- data/src/core/ext/transport/chttp2/transport/huffsyms.h +21 -21
- data/src/core/ext/transport/chttp2/transport/internal.h +248 -296
- data/src/core/ext/transport/chttp2/transport/parsing.cc +363 -132
- data/src/core/ext/transport/chttp2/transport/stream_lists.cc +30 -30
- data/src/core/ext/transport/chttp2/transport/stream_map.cc +24 -24
- data/src/core/ext/transport/chttp2/transport/stream_map.h +34 -33
- data/src/core/ext/transport/chttp2/transport/varint.cc +19 -20
- data/src/core/ext/transport/chttp2/transport/varint.h +39 -34
- data/src/core/ext/transport/chttp2/transport/writing.cc +121 -94
- data/src/core/ext/transport/inproc/inproc_plugin.cc +17 -18
- data/src/core/ext/transport/inproc/inproc_transport.cc +227 -274
- data/src/core/ext/transport/inproc/inproc_transport.h +23 -21
- data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.h +30 -10
- data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c +6 -5
- data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.h +33 -8
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c +8 -327
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.h +83 -1723
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.c +388 -0
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.h +1953 -0
- data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c +6 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.h +33 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/annotations/resource.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/annotations/resource.upb.h +7 -3
- data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c +14 -14
- data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.h +90 -30
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +25 -19
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +168 -34
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c +35 -43
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.h +195 -116
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c +11 -11
- data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.h +84 -28
- data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c +35 -12
- data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.h +179 -14
- data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c +25 -23
- data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.h +156 -48
- data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c +7 -7
- data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.h +42 -14
- data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c +12 -12
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.h +78 -26
- data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c +23 -20
- data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.h +137 -69
- data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +23 -21
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +132 -44
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c +17 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.h +67 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c +15 -9
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.h +73 -18
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.h +30 -10
- data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +39 -13
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +194 -15
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.h +42 -14
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +5 -3
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +28 -2
- data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c +7 -4
- data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.h +36 -4
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c +6 -6
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.h +48 -16
- data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.h +54 -18
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +26 -11
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +124 -14
- data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c +19 -15
- data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.h +78 -69
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +168 -82
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +887 -166
- data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c +34 -15
- data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.h +161 -22
- data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c +3 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.h +15 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c +47 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.h +107 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.c +1 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.h +1 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c +7 -6
- data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.h +27 -14
- data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.h +30 -10
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c +11 -4
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +58 -4
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c +6 -3
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.h +29 -2
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c +62 -0
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h +179 -0
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +38 -27
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +207 -52
- data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c +46 -0
- data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h +98 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c +54 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +188 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.c +111 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.h +444 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c +56 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +192 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c +46 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h +98 -0
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c +10 -8
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.h +66 -14
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c +13 -11
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +73 -23
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c +133 -16
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.h +656 -12
- data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.h +30 -10
- data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.h +24 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.c +47 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.h +113 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +24 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.c +43 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.h +114 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c +7 -7
- data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.h +42 -14
- data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.h +30 -10
- data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.h +12 -4
- data/src/core/ext/upb-generated/envoy/type/v3/range.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/type/v3/range.upb.h +18 -6
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.c +63 -0
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.h +202 -0
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_unit.upb.h +3 -1
- data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.h +6 -2
- data/src/core/ext/upb-generated/google/api/annotations.upb.h +1 -1
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c +11 -11
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.h +66 -22
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c +9 -9
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h +72 -24
- data/src/core/ext/upb-generated/google/api/http.upb.c +3 -3
- data/src/core/ext/upb-generated/google/api/http.upb.h +18 -6
- data/src/core/ext/upb-generated/google/api/httpbody.upb.c +1 -1
- data/src/core/ext/upb-generated/google/api/httpbody.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/any.upb.c +1 -1
- data/src/core/ext/upb-generated/google/protobuf/any.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c +44 -43
- data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.h +177 -55
- data/src/core/ext/upb-generated/google/protobuf/duration.upb.c +1 -1
- data/src/core/ext/upb-generated/google/protobuf/duration.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/empty.upb.c +1 -1
- data/src/core/ext/upb-generated/google/protobuf/empty.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/struct.upb.c +2 -2
- data/src/core/ext/upb-generated/google/protobuf/struct.upb.h +18 -6
- data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c +1 -1
- data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c +6 -6
- data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.h +54 -18
- data/src/core/ext/upb-generated/google/rpc/status.upb.c +1 -1
- data/src/core/ext/upb-generated/google/rpc/status.upb.h +6 -2
- data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c +2 -2
- data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.h +24 -8
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.h +6 -2
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c +7 -7
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.h +60 -20
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c +1 -1
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.h +12 -4
- data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c +2 -2
- data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.h +12 -4
- data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c +6 -6
- data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.h +54 -18
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c +2 -2
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.h +12 -4
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c +7 -7
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.h +42 -14
- data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c +3 -3
- data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.h +23 -11
- data/src/core/ext/upb-generated/udpa/annotations/security.upb.c +1 -1
- data/src/core/ext/upb-generated/udpa/annotations/security.upb.h +7 -3
- data/src/core/ext/upb-generated/udpa/annotations/status.upb.h +7 -3
- data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c +1 -1
- data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.h +7 -3
- data/src/core/ext/upb-generated/validate/validate.upb.c +22 -22
- data/src/core/ext/upb-generated/validate/validate.upb.h +139 -47
- data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c +3 -3
- data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.h +23 -11
- data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.h +7 -3
- data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c +3 -3
- data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.h +27 -11
- data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.h +7 -3
- data/src/core/ext/upb-generated/xds/core/v3/authority.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/authority.upb.h +6 -2
- data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.c +47 -0
- data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.h +107 -0
- data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c +2 -2
- data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.h +12 -4
- data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.h +6 -2
- data/src/core/ext/upb-generated/xds/core/v3/extension.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/extension.upb.h +6 -2
- data/src/core/ext/upb-generated/xds/core/v3/resource.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/resource.upb.h +6 -2
- data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.h +12 -4
- data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.h +6 -2
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +3 -2
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +15 -2
- data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c +46 -0
- data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.h +113 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.c +46 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.h +98 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.c +62 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.h +174 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.c +36 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.h +74 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.c +65 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.h +191 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c +6 -6
- data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.h +54 -18
- data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.c +127 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.h +474 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c +2 -2
- data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.h +12 -4
- data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c +1 -1
- data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.h +12 -4
- data/src/core/ext/upb-generated/xds/type/v3/cel.upb.c +67 -0
- data/src/core/ext/upb-generated/xds/type/v3/cel.upb.h +214 -0
- data/src/core/ext/upb-generated/xds/type/v3/range.upb.c +64 -0
- data/src/core/ext/upb-generated/xds/type/v3/range.upb.h +208 -0
- data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c +1 -2
- data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.h +6 -2
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/clusters.upbdefs.c +54 -53
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.c +67 -274
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.h +0 -85
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.c +277 -0
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.h +125 -0
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/listeners.upbdefs.c +14 -11
- data/src/core/ext/upbdefs-generated/envoy/config/accesslog/v3/accesslog.upbdefs.c +107 -107
- data/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +199 -187
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c +232 -222
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.h +0 -5
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.c +95 -75
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/base.upbdefs.c +187 -183
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c +81 -75
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c +190 -186
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.c +7 -3
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/substitution_format_string.upbdefs.c +16 -16
- data/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint_components.upbdefs.c +71 -66
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c +164 -137
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c +16 -12
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/udp_listener_config.upbdefs.c +39 -31
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c +140 -129
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c +82 -74
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.h +2 -2
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +740 -667
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.h +20 -0
- data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.c +150 -139
- data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/datadog.upbdefs.c +16 -14
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c +48 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/trace.upbdefs.c +16 -12
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/zipkin.upbdefs.c +20 -18
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c +46 -25
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/router/v3/router.upbdefs.c +53 -42
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c +60 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +485 -467
- data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c +50 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c +21 -18
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +149 -145
- data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.c +156 -84
- data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c +113 -113
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.c +48 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/regex.upbdefs.c +29 -28
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.c +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.c +69 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_unit.upbdefs.c +10 -9
- data/src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c +92 -90
- data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.c +45 -0
- data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.c +43 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.c +51 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.c +36 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.c +55 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/matcher.upbdefs.c +9 -9
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.c +71 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.h +60 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.c +60 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.c +36 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.h +45 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/typed_struct.upbdefs.c +4 -7
- data/src/core/ext/xds/certificate_provider_store.cc +69 -5
- data/src/core/ext/xds/certificate_provider_store.h +22 -5
- data/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +17 -10
- data/src/core/ext/xds/file_watcher_certificate_provider_factory.h +12 -4
- data/src/core/ext/xds/upb_utils.h +3 -25
- data/src/core/ext/xds/xds_api.cc +90 -196
- data/src/core/ext/xds/xds_api.h +41 -39
- data/src/core/ext/xds/xds_bootstrap.cc +5 -537
- data/src/core/ext/xds/xds_bootstrap.h +41 -96
- data/src/core/ext/xds/xds_bootstrap_grpc.cc +362 -0
- data/src/core/ext/xds/xds_bootstrap_grpc.h +184 -0
- data/src/core/ext/xds/xds_certificate_provider.cc +35 -26
- data/src/core/ext/xds/xds_certificate_provider.h +29 -5
- data/src/core/ext/xds/xds_channel_args.h +3 -3
- data/src/core/ext/xds/xds_channel_stack_modifier.cc +17 -7
- data/src/core/ext/xds/xds_channel_stack_modifier.h +16 -4
- data/src/core/ext/xds/xds_client.cc +849 -1316
- data/src/core/ext/xds/xds_client.h +57 -61
- data/src/core/ext/xds/xds_client_grpc.cc +235 -0
- data/src/core/ext/xds/xds_client_grpc.h +79 -0
- data/src/core/ext/xds/xds_client_stats.cc +24 -25
- data/src/core/ext/xds/xds_client_stats.h +26 -24
- data/src/core/ext/xds/xds_cluster.cc +505 -247
- data/src/core/ext/xds/xds_cluster.h +69 -40
- data/src/core/ext/xds/xds_cluster_specifier_plugin.cc +67 -77
- data/src/core/ext/xds/xds_cluster_specifier_plugin.h +47 -29
- data/src/core/ext/xds/xds_common_types.cc +238 -123
- data/src/core/ext/xds/xds_common_types.h +29 -16
- data/src/core/ext/xds/xds_endpoint.cc +265 -162
- data/src/core/ext/xds/xds_endpoint.h +19 -15
- data/src/core/ext/xds/xds_health_status.cc +80 -0
- data/src/core/ext/xds/xds_health_status.h +109 -0
- data/src/core/ext/xds/xds_http_fault_filter.cc +67 -58
- data/src/core/ext/xds/xds_http_fault_filter.h +19 -25
- data/src/core/ext/xds/xds_http_filters.cc +70 -71
- data/src/core/ext/xds/xds_http_filters.h +73 -25
- data/src/core/ext/xds/xds_http_rbac_filter.cc +168 -225
- data/src/core/ext/xds/xds_http_rbac_filter.h +22 -18
- data/src/core/ext/xds/xds_http_stateful_session_filter.cc +218 -0
- data/src/core/ext/xds/xds_http_stateful_session_filter.h +58 -0
- data/src/core/ext/xds/xds_lb_policy_registry.cc +334 -0
- data/src/core/ext/xds/xds_lb_policy_registry.h +71 -0
- data/src/core/ext/xds/xds_listener.cc +573 -481
- data/src/core/ext/xds/xds_listener.h +57 -51
- data/src/core/ext/xds/xds_resource_type.h +28 -22
- data/src/core/ext/xds/xds_resource_type_impl.h +18 -17
- data/src/core/ext/xds/xds_route_config.cc +592 -570
- data/src/core/ext/xds/xds_route_config.h +74 -40
- data/src/core/ext/xds/xds_routing.cc +21 -7
- data/src/core/ext/xds/xds_routing.h +17 -12
- data/src/core/ext/xds/xds_server_config_fetcher.cc +227 -178
- data/src/core/ext/xds/xds_transport.h +86 -0
- data/src/core/ext/xds/xds_transport_grpc.cc +356 -0
- data/src/core/ext/xds/xds_transport_grpc.h +135 -0
- data/src/core/lib/address_utils/parse_address.cc +52 -52
- data/src/core/lib/address_utils/parse_address.h +46 -42
- data/src/core/lib/address_utils/sockaddr_utils.cc +90 -54
- data/src/core/lib/address_utils/sockaddr_utils.h +45 -40
- data/src/core/lib/avl/avl.h +57 -33
- data/src/core/lib/backoff/backoff.cc +19 -21
- data/src/core/lib/backoff/backoff.h +21 -21
- data/src/core/lib/backoff/random_early_detection.cc +31 -0
- data/src/core/lib/backoff/random_early_detection.h +59 -0
- data/src/core/lib/channel/call_finalization.h +6 -4
- data/src/core/lib/channel/call_tracer.h +23 -8
- data/src/core/lib/channel/channel_args.cc +149 -61
- data/src/core/lib/channel/channel_args.h +308 -104
- data/src/core/lib/channel/channel_args_preconditioning.cc +4 -3
- data/src/core/lib/channel/channel_args_preconditioning.h +6 -5
- data/src/core/lib/channel/channel_fwd.h +26 -0
- data/src/core/lib/channel/channel_stack.cc +92 -52
- data/src/core/lib/channel/channel_stack.h +184 -135
- data/src/core/lib/channel/channel_stack_builder.cc +26 -39
- data/src/core/lib/channel/channel_stack_builder.h +33 -40
- data/src/core/lib/channel/channel_stack_builder_impl.cc +84 -60
- data/src/core/lib/channel/channel_stack_builder_impl.h +9 -9
- data/src/core/lib/channel/channel_trace.cc +29 -36
- data/src/core/lib/channel/channel_trace.h +27 -24
- data/src/core/lib/channel/channelz.cc +50 -65
- data/src/core/lib/channel/channelz.h +50 -33
- data/src/core/lib/channel/channelz_registry.cc +28 -36
- data/src/core/lib/channel/channelz_registry.h +31 -30
- data/src/core/lib/channel/connected_channel.cc +1267 -78
- data/src/core/lib/channel/connected_channel.h +21 -23
- data/src/core/lib/channel/context.h +25 -21
- data/src/core/lib/channel/promise_based_filter.cc +1896 -377
- data/src/core/lib/channel/promise_based_filter.h +592 -92
- data/src/core/lib/channel/status_util.cc +64 -17
- data/src/core/lib/channel/status_util.h +38 -24
- data/src/core/lib/compression/compression.cc +24 -19
- data/src/core/lib/compression/compression_internal.cc +88 -63
- data/src/core/lib/compression/compression_internal.h +28 -27
- data/src/core/lib/compression/message_compress.cc +28 -26
- data/src/core/lib/compression/message_compress.h +28 -29
- data/src/core/lib/config/core_configuration.cc +8 -1
- data/src/core/lib/config/core_configuration.h +86 -39
- data/src/core/lib/debug/event_log.cc +88 -0
- data/src/core/lib/debug/event_log.h +81 -0
- data/src/core/lib/debug/histogram_view.cc +69 -0
- data/src/core/lib/debug/histogram_view.h +37 -0
- data/src/core/lib/debug/stats.cc +48 -152
- data/src/core/lib/debug/stats.h +50 -57
- data/src/core/lib/debug/stats_data.cc +302 -645
- data/src/core/lib/debug/stats_data.h +293 -545
- data/src/core/lib/debug/trace.cc +18 -20
- data/src/core/lib/debug/trace.h +27 -48
- data/src/core/lib/event_engine/channel_args_endpoint_config.cc +16 -17
- data/src/core/lib/event_engine/channel_args_endpoint_config.h +16 -9
- data/src/core/lib/event_engine/common_closures.h +71 -0
- data/src/core/lib/event_engine/default_event_engine.cc +99 -0
- data/src/core/lib/event_engine/default_event_engine.h +73 -0
- data/src/core/lib/event_engine/default_event_engine_factory.cc +24 -3
- data/src/core/lib/event_engine/{event_engine_factory.h → default_event_engine_factory.h} +6 -9
- data/src/core/lib/event_engine/event_engine.cc +3 -30
- data/src/core/lib/event_engine/executor/executor.h +38 -0
- data/src/core/lib/event_engine/forkable.cc +106 -0
- data/src/core/lib/event_engine/forkable.h +61 -0
- data/src/core/lib/event_engine/handle_containers.h +74 -0
- data/src/core/lib/event_engine/memory_allocator.cc +11 -3
- data/src/core/lib/event_engine/poller.h +62 -0
- data/src/core/lib/event_engine/posix.h +158 -0
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +642 -0
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.h +139 -0
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +899 -0
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.h +97 -0
- data/src/core/lib/event_engine/posix_engine/event_poller.h +111 -0
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +74 -0
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.h +33 -0
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +77 -0
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.h +179 -0
- data/src/core/lib/event_engine/posix_engine/lockfree_event.cc +265 -0
- data/src/core/lib/event_engine/posix_engine/lockfree_event.h +73 -0
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +1305 -0
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +717 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +640 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine.h +259 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_closure.h +80 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +289 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.h +279 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +379 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.h +91 -0
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +853 -0
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +316 -0
- data/src/core/lib/event_engine/posix_engine/timer.cc +311 -0
- data/src/core/lib/event_engine/posix_engine/timer.h +194 -0
- data/src/core/lib/event_engine/posix_engine/timer_heap.cc +107 -0
- data/src/core/lib/event_engine/posix_engine/timer_heap.h +56 -0
- data/src/core/lib/event_engine/posix_engine/timer_manager.cc +173 -0
- data/src/core/lib/event_engine/posix_engine/timer_manager.h +114 -0
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +332 -0
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.h +185 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc +127 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.h +45 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc +150 -0
- data/src/core/lib/{iomgr/event_engine/promise.h → event_engine/posix_engine/wakeup_fd_pipe.h} +19 -25
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h +76 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc +67 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h +37 -0
- data/src/core/lib/event_engine/resolved_address.cc +23 -2
- data/src/core/lib/{iomgr/event_engine → event_engine}/resolved_address_internal.h +5 -8
- data/src/core/lib/event_engine/shim.cc +56 -0
- data/src/core/lib/event_engine/shim.h +33 -0
- data/src/core/lib/event_engine/slice.cc +103 -0
- data/src/core/lib/event_engine/slice_buffer.cc +50 -0
- data/src/core/lib/event_engine/tcp_socket_utils.cc +389 -0
- data/src/core/lib/event_engine/tcp_socket_utils.h +90 -0
- data/src/core/lib/event_engine/thread_local.cc +29 -0
- data/src/core/lib/event_engine/thread_local.h +32 -0
- data/src/core/lib/event_engine/thread_pool.cc +253 -0
- data/src/core/lib/event_engine/thread_pool.h +141 -0
- data/src/core/lib/event_engine/time_util.cc +30 -0
- data/src/core/lib/event_engine/time_util.h +32 -0
- data/src/core/lib/{iomgr/endpoint_pair_event_engine.cc → event_engine/trace.cc} +9 -17
- data/src/core/lib/event_engine/trace.h +43 -0
- data/src/core/lib/event_engine/utils.cc +44 -0
- data/src/core/lib/event_engine/utils.h +44 -0
- data/src/core/lib/event_engine/windows/iocp.cc +140 -0
- data/src/core/lib/event_engine/windows/iocp.h +69 -0
- data/src/core/lib/event_engine/windows/win_socket.cc +219 -0
- data/src/core/lib/event_engine/windows/win_socket.h +129 -0
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +331 -0
- data/src/core/lib/event_engine/windows/windows_endpoint.h +103 -0
- data/src/core/lib/event_engine/windows/windows_engine.cc +388 -0
- data/src/core/lib/event_engine/windows/windows_engine.h +163 -0
- data/src/core/lib/experiments/config.cc +161 -0
- data/src/core/lib/experiments/config.h +53 -0
- data/src/core/lib/experiments/experiments.cc +81 -0
- data/src/core/lib/experiments/experiments.h +117 -0
- data/src/core/lib/gpr/alloc.cc +19 -25
- data/src/core/lib/gpr/alloc.h +20 -20
- data/src/core/lib/gpr/atm.cc +17 -17
- data/src/core/lib/gpr/cpu_iphone.cc +24 -24
- data/src/core/lib/gpr/cpu_linux.cc +28 -23
- data/src/core/lib/gpr/cpu_posix.cc +23 -22
- data/src/core/lib/gpr/cpu_windows.cc +20 -18
- data/src/core/lib/gpr/log.cc +30 -17
- data/src/core/lib/gpr/log_android.cc +22 -20
- data/src/core/lib/gpr/log_linux.cc +24 -24
- data/src/core/lib/gpr/log_posix.cc +20 -19
- data/src/core/lib/gpr/log_windows.cc +25 -25
- data/src/core/lib/gpr/spinlock.h +20 -20
- data/src/core/lib/gpr/string.cc +25 -24
- data/src/core/lib/gpr/string.h +61 -61
- data/src/core/lib/gpr/string_posix.cc +24 -24
- data/src/core/lib/gpr/string_util_windows.cc +25 -52
- data/src/core/lib/gpr/string_windows.cc +24 -24
- data/src/core/lib/gpr/sync.cc +25 -25
- data/src/core/lib/gpr/sync_abseil.cc +36 -40
- data/src/core/lib/gpr/sync_posix.cc +22 -34
- data/src/core/lib/gpr/sync_windows.cc +29 -27
- data/src/core/lib/gpr/time.cc +34 -30
- data/src/core/lib/gpr/time_posix.cc +41 -45
- data/src/core/lib/gpr/time_precise.cc +22 -22
- data/src/core/lib/gpr/time_precise.h +21 -22
- data/src/core/lib/gpr/time_windows.cc +35 -29
- data/src/core/lib/gpr/tmpfile.h +24 -24
- data/src/core/lib/gpr/tmpfile_msys.cc +21 -20
- data/src/core/lib/gpr/tmpfile_posix.cc +22 -20
- data/src/core/lib/gpr/tmpfile_windows.cc +28 -29
- data/src/core/lib/gpr/useful.h +83 -30
- data/src/core/lib/gpr/wrap_memcpy.cc +23 -23
- data/src/core/lib/gprpp/atomic_utils.h +20 -20
- data/src/core/lib/gprpp/bitset.h +35 -17
- data/src/core/lib/gprpp/chunked_vector.h +7 -3
- data/src/core/lib/gprpp/construct_destruct.h +4 -3
- data/src/core/lib/gprpp/cpp_impl_of.h +3 -3
- data/src/core/{ext/xds/xds_resource_type.cc → lib/gprpp/crash.cc} +12 -12
- data/src/core/lib/gprpp/crash.h +34 -0
- data/src/core/lib/gprpp/debug_location.h +60 -31
- data/src/core/lib/gprpp/dual_ref_counted.h +4 -7
- data/src/core/lib/gprpp/env.h +53 -0
- data/src/core/lib/gprpp/env_linux.cc +80 -0
- data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_event_engine.cc → lib/gprpp/env_posix.cc} +25 -9
- data/src/core/lib/gprpp/env_windows.cc +56 -0
- data/src/core/lib/gprpp/examine_stack.cc +17 -17
- data/src/core/lib/gprpp/examine_stack.h +21 -22
- data/src/core/lib/gprpp/fork.cc +58 -53
- data/src/core/lib/gprpp/fork.h +29 -35
- data/src/core/lib/gprpp/global_config.h +22 -24
- data/src/core/lib/gprpp/global_config_custom.h +20 -20
- data/src/core/lib/gprpp/global_config_env.cc +28 -25
- data/src/core/lib/gprpp/global_config_env.h +29 -27
- data/src/core/lib/gprpp/global_config_generic.h +21 -25
- data/src/core/lib/gprpp/host_port.cc +30 -26
- data/src/core/lib/gprpp/host_port.h +32 -31
- data/src/core/lib/gprpp/load_file.cc +75 -0
- data/src/core/lib/gprpp/load_file.h +33 -0
- data/src/core/lib/gprpp/manual_constructor.h +21 -89
- data/src/core/lib/gprpp/match.h +5 -3
- data/src/core/lib/gprpp/memory.h +22 -26
- data/src/core/lib/gprpp/mpscq.cc +17 -17
- data/src/core/lib/gprpp/mpscq.h +21 -21
- data/src/core/lib/gprpp/no_destruct.h +95 -0
- data/src/core/lib/gprpp/notification.h +67 -0
- data/src/core/lib/gprpp/orphanable.h +22 -25
- data/src/core/lib/gprpp/overload.h +3 -3
- data/src/core/lib/gprpp/packed_table.h +40 -0
- data/src/core/lib/gprpp/per_cpu.h +46 -0
- data/src/core/lib/gprpp/ref_counted.h +22 -24
- data/src/core/lib/gprpp/ref_counted_ptr.h +42 -55
- data/src/core/lib/gprpp/single_set_ptr.h +3 -3
- data/src/core/lib/gprpp/sorted_pack.h +98 -0
- data/src/core/lib/gprpp/stat.h +3 -5
- data/src/core/lib/gprpp/stat_posix.cc +10 -4
- data/src/core/lib/gprpp/stat_windows.cc +4 -2
- data/src/core/lib/gprpp/status_helper.cc +55 -35
- data/src/core/lib/gprpp/status_helper.h +13 -5
- data/src/core/lib/gprpp/strerror.cc +43 -0
- data/src/core/lib/gprpp/strerror.h +29 -0
- data/src/core/lib/gprpp/sync.h +26 -24
- data/src/core/lib/gprpp/table.h +22 -4
- data/src/core/lib/gprpp/tchar.cc +49 -0
- data/src/core/lib/gprpp/tchar.h +33 -0
- data/src/core/lib/gprpp/thd.h +25 -28
- data/src/core/lib/gprpp/thd_posix.cc +33 -32
- data/src/core/lib/gprpp/thd_windows.cc +28 -26
- data/src/core/lib/gprpp/time.cc +52 -9
- data/src/core/lib/gprpp/time.h +78 -4
- data/src/core/lib/gprpp/time_averaged_stats.cc +60 -0
- data/src/core/lib/gprpp/time_averaged_stats.h +79 -0
- data/src/core/lib/gprpp/time_util.cc +4 -0
- data/src/core/lib/gprpp/time_util.h +3 -3
- data/src/core/lib/gprpp/unique_type_name.h +104 -0
- data/src/core/lib/gprpp/validation_errors.cc +61 -0
- data/src/core/lib/gprpp/validation_errors.h +127 -0
- data/src/core/lib/{iomgr → gprpp}/work_serializer.cc +34 -18
- data/src/core/lib/{iomgr → gprpp}/work_serializer.h +21 -28
- data/src/core/lib/handshaker/proxy_mapper.h +53 -0
- data/src/core/lib/handshaker/proxy_mapper_registry.cc +71 -0
- data/src/core/lib/handshaker/proxy_mapper_registry.h +75 -0
- data/src/core/lib/http/format_request.cc +53 -22
- data/src/core/lib/http/format_request.h +23 -21
- data/src/core/lib/http/httpcli.cc +139 -141
- data/src/core/lib/http/httpcli.h +86 -38
- data/src/core/lib/http/httpcli_security_connector.cc +50 -42
- data/src/core/lib/http/httpcli_ssl_credentials.h +6 -4
- data/src/core/lib/http/parser.cc +78 -90
- data/src/core/lib/http/parser.h +38 -35
- data/src/core/lib/iomgr/block_annotate.h +23 -23
- data/src/core/lib/iomgr/buffer_list.cc +156 -136
- data/src/core/lib/iomgr/buffer_list.h +123 -101
- data/src/core/lib/iomgr/call_combiner.cc +32 -64
- data/src/core/lib/iomgr/call_combiner.h +24 -25
- data/src/core/lib/iomgr/cfstream_handle.cc +33 -36
- data/src/core/lib/iomgr/cfstream_handle.h +25 -25
- data/src/core/lib/iomgr/{event_engine/pollset.h → closure.cc} +9 -7
- data/src/core/lib/iomgr/closure.h +95 -58
- data/src/core/lib/iomgr/combiner.cc +20 -39
- data/src/core/lib/iomgr/combiner.h +20 -20
- data/src/core/lib/iomgr/dualstack_socket_posix.cc +21 -21
- data/src/core/lib/iomgr/dynamic_annotations.h +22 -22
- data/src/core/lib/iomgr/endpoint.cc +21 -21
- data/src/core/lib/iomgr/endpoint.h +53 -51
- data/src/core/lib/iomgr/endpoint_cfstream.cc +51 -47
- data/src/core/lib/iomgr/endpoint_cfstream.h +32 -32
- data/src/core/lib/iomgr/endpoint_pair.h +22 -22
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +36 -29
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +31 -22
- data/src/core/lib/iomgr/error.cc +51 -834
- data/src/core/lib/iomgr/error.h +45 -317
- data/src/core/lib/iomgr/error_cfstream.cc +18 -23
- data/src/core/lib/iomgr/error_cfstream.h +21 -21
- data/src/core/lib/iomgr/ev_apple.cc +21 -21
- data/src/core/lib/iomgr/ev_apple.h +21 -21
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +263 -248
- data/src/core/lib/iomgr/ev_epoll1_linux.h +21 -21
- data/src/core/lib/iomgr/ev_poll_posix.cc +296 -271
- data/src/core/lib/iomgr/ev_poll_posix.h +22 -22
- data/src/core/lib/iomgr/ev_posix.cc +88 -121
- data/src/core/lib/iomgr/ev_posix.h +93 -87
- data/src/core/lib/iomgr/ev_windows.cc +18 -18
- data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.cc +15 -30
- data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.h +10 -13
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +430 -0
- data/src/core/lib/iomgr/event_engine_shims/endpoint.h +43 -0
- data/src/core/lib/iomgr/event_engine_shims/tcp_client.cc +91 -0
- data/src/core/lib/iomgr/event_engine_shims/tcp_client.h +44 -0
- data/src/core/lib/iomgr/exec_ctx.cc +34 -56
- data/src/core/lib/iomgr/exec_ctx.h +152 -177
- data/src/core/lib/iomgr/executor.cc +21 -31
- data/src/core/lib/iomgr/executor.h +27 -30
- data/src/core/lib/iomgr/fork_posix.cc +30 -27
- data/src/core/lib/iomgr/fork_windows.cc +21 -21
- data/src/core/lib/iomgr/gethostname.h +20 -20
- data/src/core/lib/iomgr/gethostname_fallback.cc +17 -17
- data/src/core/lib/iomgr/gethostname_host_name_max.cc +17 -17
- data/src/core/lib/iomgr/gethostname_sysconf.cc +17 -17
- data/src/core/lib/iomgr/grpc_if_nametoindex.h +22 -22
- data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +20 -19
- data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +20 -19
- data/src/core/lib/iomgr/internal_errqueue.cc +41 -48
- data/src/core/lib/iomgr/internal_errqueue.h +84 -89
- data/src/core/lib/iomgr/iocp_windows.cc +23 -23
- data/src/core/lib/iomgr/iocp_windows.h +21 -21
- data/src/core/lib/iomgr/iomgr.cc +25 -20
- data/src/core/lib/iomgr/iomgr.h +35 -35
- data/src/core/lib/iomgr/iomgr_fwd.h +26 -0
- data/src/core/lib/iomgr/iomgr_internal.cc +17 -17
- data/src/core/lib/iomgr/iomgr_internal.h +28 -28
- data/src/core/lib/iomgr/iomgr_posix.cc +21 -20
- data/src/core/lib/iomgr/iomgr_posix_cfstream.cc +21 -19
- data/src/core/lib/iomgr/iomgr_windows.cc +24 -22
- data/src/core/lib/iomgr/load_file.cc +24 -27
- data/src/core/lib/iomgr/load_file.h +22 -22
- data/src/core/lib/iomgr/lockfree_event.cc +114 -131
- data/src/core/lib/iomgr/lockfree_event.h +23 -23
- data/src/core/lib/iomgr/nameser.h +86 -86
- data/src/core/lib/iomgr/polling_entity.cc +25 -21
- data/src/core/lib/iomgr/polling_entity.h +29 -29
- data/src/core/lib/iomgr/pollset.cc +17 -17
- data/src/core/lib/iomgr/pollset.h +55 -55
- data/src/core/lib/iomgr/pollset_set.cc +17 -17
- data/src/core/lib/iomgr/pollset_set.h +26 -28
- data/src/core/lib/iomgr/pollset_set_windows.cc +18 -18
- data/src/core/lib/iomgr/pollset_set_windows.h +20 -20
- data/src/core/lib/iomgr/pollset_windows.cc +32 -31
- data/src/core/lib/iomgr/pollset_windows.h +24 -24
- data/src/core/lib/iomgr/port.h +59 -37
- data/src/core/lib/iomgr/python_util.h +24 -24
- data/src/core/lib/iomgr/resolve_address.cc +34 -20
- data/src/core/lib/iomgr/resolve_address.h +73 -43
- data/src/core/lib/iomgr/resolve_address_impl.h +7 -7
- data/src/core/lib/iomgr/resolve_address_posix.cc +81 -62
- data/src/core/lib/iomgr/resolve_address_posix.h +26 -10
- data/src/core/lib/iomgr/resolve_address_windows.cc +64 -38
- data/src/core/lib/iomgr/resolve_address_windows.h +26 -10
- data/src/core/lib/iomgr/resolved_address.h +3 -3
- data/src/core/lib/iomgr/sockaddr.h +25 -26
- data/src/core/lib/iomgr/sockaddr_posix.h +23 -21
- data/src/core/lib/iomgr/sockaddr_utils_posix.cc +20 -18
- data/src/core/lib/iomgr/sockaddr_windows.h +23 -21
- data/src/core/lib/iomgr/socket_factory_posix.cc +18 -18
- data/src/core/lib/iomgr/socket_factory_posix.h +32 -32
- data/src/core/lib/iomgr/socket_mutator.cc +19 -18
- data/src/core/lib/iomgr/socket_mutator.h +39 -39
- data/src/core/lib/iomgr/socket_utils.h +27 -27
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +82 -101
- data/src/core/lib/iomgr/socket_utils_linux.cc +18 -17
- data/src/core/lib/iomgr/socket_utils_posix.cc +103 -19
- data/src/core/lib/iomgr/socket_utils_posix.h +176 -84
- data/src/core/lib/iomgr/socket_utils_windows.cc +20 -19
- data/src/core/lib/iomgr/socket_windows.cc +37 -36
- data/src/core/lib/iomgr/socket_windows.h +59 -61
- data/src/core/lib/iomgr/systemd_utils.cc +116 -0
- data/src/core/lib/iomgr/systemd_utils.h +33 -0
- data/src/core/lib/iomgr/tcp_client.cc +28 -24
- data/src/core/lib/iomgr/tcp_client.h +49 -38
- data/src/core/lib/iomgr/tcp_client_cfstream.cc +46 -35
- data/src/core/lib/iomgr/tcp_client_posix.cc +243 -105
- data/src/core/lib/iomgr/tcp_client_posix.h +55 -52
- data/src/core/lib/iomgr/tcp_client_windows.cc +73 -53
- data/src/core/lib/iomgr/tcp_posix.cc +505 -362
- data/src/core/lib/iomgr/tcp_posix.h +32 -30
- data/src/core/lib/iomgr/tcp_server.cc +33 -24
- data/src/core/lib/iomgr/tcp_server.h +78 -69
- data/src/core/lib/iomgr/tcp_server_posix.cc +381 -133
- data/src/core/lib/iomgr/tcp_server_utils_posix.h +68 -55
- data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +120 -82
- data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +46 -43
- data/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc +19 -19
- data/src/core/lib/iomgr/tcp_server_windows.cc +116 -118
- data/src/core/lib/iomgr/tcp_windows.cc +116 -98
- data/src/core/lib/iomgr/tcp_windows.h +34 -35
- data/src/core/lib/iomgr/timer.cc +17 -17
- data/src/core/lib/iomgr/timer.h +68 -68
- data/src/core/lib/iomgr/timer_generic.cc +125 -134
- data/src/core/lib/iomgr/timer_generic.h +21 -21
- data/src/core/lib/iomgr/timer_heap.cc +25 -25
- data/src/core/lib/iomgr/timer_heap.h +22 -22
- data/src/core/lib/iomgr/timer_manager.cc +31 -31
- data/src/core/lib/iomgr/timer_manager.h +27 -27
- data/src/core/lib/iomgr/unix_sockets_posix.cc +20 -21
- data/src/core/lib/iomgr/unix_sockets_posix.h +21 -21
- data/src/core/lib/iomgr/unix_sockets_posix_noop.cc +19 -17
- data/src/core/lib/iomgr/wakeup_fd_eventfd.cc +23 -23
- data/src/core/lib/iomgr/wakeup_fd_nospecial.cc +23 -23
- data/src/core/lib/iomgr/wakeup_fd_pipe.cc +29 -26
- data/src/core/lib/iomgr/wakeup_fd_pipe.h +20 -20
- data/src/core/lib/iomgr/wakeup_fd_posix.cc +33 -30
- data/src/core/lib/iomgr/wakeup_fd_posix.h +52 -54
- data/src/core/lib/json/json.h +23 -27
- data/src/core/{ext/filters/http/message_compress/message_decompress_filter.h → lib/json/json_args.h} +13 -10
- data/src/core/lib/json/json_channel_args.h +42 -0
- data/src/core/lib/json/json_object_loader.cc +217 -0
- data/src/core/lib/json/json_object_loader.h +634 -0
- data/src/core/lib/json/json_reader.cc +123 -90
- data/src/core/lib/json/json_util.cc +14 -34
- data/src/core/lib/json/json_util.h +17 -8
- data/src/core/lib/json/json_writer.cc +62 -57
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.cc +30 -60
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.h +47 -62
- data/src/core/lib/load_balancing/lb_policy_factory.h +49 -0
- data/src/core/lib/load_balancing/lb_policy_registry.cc +142 -0
- data/src/core/lib/load_balancing/lb_policy_registry.h +82 -0
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/subchannel_interface.h +57 -52
- data/src/core/lib/matchers/matchers.cc +12 -7
- data/src/core/lib/matchers/matchers.h +5 -3
- data/src/core/lib/promise/activity.cc +23 -8
- data/src/core/lib/promise/activity.h +116 -58
- data/src/core/lib/promise/arena_promise.h +115 -72
- data/src/core/lib/promise/context.h +18 -11
- data/src/core/lib/promise/detail/basic_join.h +197 -0
- data/src/core/lib/promise/detail/basic_seq.h +28 -33
- data/src/core/lib/promise/detail/promise_factory.h +61 -14
- data/src/core/lib/promise/detail/promise_like.h +3 -3
- data/src/core/lib/promise/detail/status.h +31 -3
- data/src/core/lib/promise/detail/switch.h +21 -21
- data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h +23 -15
- data/src/core/lib/promise/if.h +195 -0
- data/src/core/lib/promise/interceptor_list.h +308 -0
- data/src/core/lib/promise/intra_activity_waiter.h +9 -3
- data/src/core/lib/promise/latch.h +109 -22
- data/src/core/lib/promise/loop.h +13 -9
- data/src/core/lib/promise/map.h +4 -6
- data/src/core/lib/promise/pipe.h +608 -0
- data/src/core/lib/promise/poll.h +129 -11
- data/src/core/lib/promise/promise.h +5 -5
- data/src/core/lib/promise/race.h +6 -10
- data/src/core/lib/promise/seq.h +32 -14
- data/src/core/lib/promise/sleep.cc +58 -42
- data/src/core/lib/promise/sleep.h +44 -26
- data/src/core/lib/promise/trace.cc +20 -0
- data/src/core/lib/promise/trace.h +24 -0
- data/src/core/lib/promise/try_join.h +82 -0
- data/src/core/lib/promise/try_seq.h +41 -23
- data/src/core/lib/resolver/resolver.cc +17 -59
- data/src/core/lib/resolver/resolver.h +21 -18
- data/src/core/lib/resolver/resolver_factory.h +11 -10
- data/src/core/lib/resolver/resolver_registry.cc +17 -9
- data/src/core/lib/resolver/resolver_registry.h +15 -5
- data/src/core/lib/resolver/server_address.cc +46 -34
- data/src/core/lib/resolver/server_address.h +36 -35
- data/src/core/lib/resource_quota/api.cc +23 -2
- data/src/core/lib/resource_quota/api.h +13 -4
- data/src/core/lib/resource_quota/arena.cc +85 -27
- data/src/core/lib/resource_quota/arena.h +197 -25
- data/src/core/lib/resource_quota/memory_quota.cc +322 -101
- data/src/core/lib/resource_quota/memory_quota.h +191 -56
- data/src/core/lib/resource_quota/periodic_update.cc +78 -0
- data/src/core/lib/resource_quota/periodic_update.h +71 -0
- data/src/core/lib/resource_quota/resource_quota.h +12 -4
- data/src/core/lib/resource_quota/thread_quota.cc +2 -0
- data/src/core/lib/resource_quota/thread_quota.h +7 -3
- data/src/core/lib/resource_quota/trace.h +3 -3
- data/src/core/lib/security/authorization/authorization_engine.h +3 -3
- data/src/core/lib/security/authorization/authorization_policy_provider.h +10 -3
- data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +3 -0
- data/src/core/lib/security/authorization/evaluate_args.cc +17 -8
- data/src/core/lib/security/authorization/evaluate_args.h +9 -6
- data/src/core/lib/security/authorization/grpc_authorization_engine.cc +5 -1
- data/src/core/lib/security/authorization/grpc_authorization_engine.h +10 -3
- data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +19 -3
- data/src/core/lib/security/authorization/grpc_server_authz_filter.h +16 -5
- data/src/core/lib/security/authorization/matchers.cc +44 -30
- data/src/core/lib/security/authorization/matchers.h +10 -3
- data/src/core/lib/security/authorization/rbac_policy.cc +6 -2
- data/src/core/lib/security/authorization/rbac_policy.h +10 -3
- data/src/core/{ext/xds → lib/security/certificate_provider}/certificate_provider_factory.h +9 -4
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +60 -0
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.h +70 -0
- data/src/core/lib/security/context/security_context.cc +27 -25
- data/src/core/lib/security/context/security_context.h +56 -34
- data/src/core/lib/security/credentials/alts/alts_credentials.cc +31 -24
- data/src/core/lib/security/credentials/alts/alts_credentials.h +63 -59
- data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +19 -19
- data/src/core/lib/security/credentials/alts/check_gcp_environment.h +43 -43
- data/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +17 -17
- data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +18 -17
- data/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc +18 -17
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +20 -22
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +18 -18
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h +38 -39
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc +19 -22
- data/src/core/lib/security/credentials/call_creds_util.cc +10 -0
- data/src/core/lib/security/credentials/call_creds_util.h +4 -3
- data/src/core/lib/security/credentials/channel_creds_registry.h +10 -4
- data/src/core/lib/security/credentials/channel_creds_registry_init.cc +13 -3
- data/src/core/lib/security/credentials/composite/composite_credentials.cc +34 -34
- data/src/core/lib/security/credentials/composite/composite_credentials.h +45 -33
- data/src/core/lib/security/credentials/credentials.cc +23 -26
- data/src/core/lib/security/credentials/credentials.h +79 -65
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +202 -74
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +21 -3
- data/src/core/lib/security/credentials/external/aws_request_signer.cc +14 -3
- data/src/core/lib/security/credentials/external/aws_request_signer.h +3 -3
- data/src/core/lib/security/credentials/external/external_account_credentials.cc +86 -68
- data/src/core/lib/security/credentials/external/external_account_credentials.h +14 -3
- data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +29 -26
- data/src/core/lib/security/credentials/external/file_external_account_credentials.h +9 -3
- data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +44 -31
- data/src/core/lib/security/credentials/external/url_external_account_credentials.h +13 -3
- data/src/core/lib/security/credentials/fake/fake_credentials.cc +43 -39
- data/src/core/lib/security/credentials/fake/fake_credentials.h +51 -44
- data/src/core/lib/security/credentials/google_default/credentials_generic.cc +23 -25
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +146 -114
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +33 -27
- data/src/core/lib/security/credentials/iam/iam_credentials.cc +31 -21
- data/src/core/lib/security/credentials/iam/iam_credentials.h +33 -23
- data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +15 -5
- data/src/core/lib/security/credentials/insecure/insecure_credentials.h +14 -10
- data/src/core/lib/security/credentials/jwt/json_token.cc +40 -29
- data/src/core/lib/security/credentials/jwt/json_token.h +36 -36
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +32 -29
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +37 -23
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +113 -89
- data/src/core/lib/security/credentials/jwt/jwt_verifier.h +49 -47
- data/src/core/lib/security/credentials/local/local_credentials.cc +32 -26
- data/src/core/lib/security/credentials/local/local_credentials.h +34 -28
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +95 -87
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +46 -25
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +35 -31
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +48 -24
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +44 -42
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +39 -27
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc +22 -30
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h +16 -17
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +65 -48
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +27 -16
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +36 -4
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +37 -14
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +22 -23
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +3 -3
- data/src/core/lib/security/credentials/tls/tls_credentials.cc +46 -44
- data/src/core/lib/security/credentials/tls/tls_credentials.h +30 -27
- data/src/core/lib/security/credentials/tls/tls_utils.cc +5 -1
- data/src/core/lib/security/credentials/tls/tls_utils.h +4 -4
- data/src/core/lib/security/credentials/xds/xds_credentials.cc +33 -35
- data/src/core/lib/security/credentials/xds/xds_credentials.h +24 -10
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +60 -54
- data/src/core/lib/security/security_connector/alts/alts_security_connector.h +49 -47
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +75 -79
- data/src/core/lib/security/security_connector/fake/fake_security_connector.h +25 -26
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc +15 -6
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.h +26 -15
- data/src/core/lib/security/security_connector/load_system_roots.h +20 -20
- data/src/core/lib/security/security_connector/load_system_roots_fallback.cc +22 -20
- data/src/core/lib/security/security_connector/{load_system_roots_linux.cc → load_system_roots_supported.cc} +43 -38
- data/src/core/lib/security/security_connector/load_system_roots_supported.h +45 -0
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +64 -47
- data/src/core/lib/security/security_connector/local/local_security_connector.h +49 -46
- data/src/core/lib/security/security_connector/security_connector.cc +37 -35
- data/src/core/lib/security/security_connector/security_connector.h +58 -45
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +55 -46
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +44 -43
- data/src/core/lib/security/security_connector/ssl_utils.cc +47 -39
- data/src/core/lib/security/security_connector/ssl_utils.h +45 -42
- data/src/core/lib/security/security_connector/ssl_utils_config.cc +22 -22
- data/src/core/lib/security/security_connector/ssl_utils_config.h +21 -22
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +58 -56
- data/src/core/lib/security/security_connector/tls/tls_security_connector.h +48 -28
- data/src/core/lib/security/transport/auth_filters.h +53 -24
- data/src/core/lib/security/transport/client_auth_filter.cc +57 -40
- data/src/core/lib/security/transport/secure_endpoint.cc +130 -75
- data/src/core/lib/security/transport/secure_endpoint.h +28 -27
- data/src/core/lib/security/transport/security_handshaker.cc +144 -114
- data/src/core/lib/security/transport/security_handshaker.h +28 -23
- data/src/core/lib/security/transport/server_auth_filter.cc +157 -266
- data/src/core/lib/security/transport/tsi_error.cc +23 -20
- data/src/core/lib/security/transport/tsi_error.h +20 -20
- data/src/core/lib/security/util/json_util.cc +24 -24
- data/src/core/lib/security/util/json_util.h +21 -23
- data/src/core/lib/service_config/service_config.h +19 -12
- data/src/core/lib/service_config/service_config_call_data.h +15 -5
- data/src/core/lib/service_config/service_config_impl.cc +111 -150
- data/src/core/lib/service_config/service_config_impl.h +27 -27
- data/src/core/lib/service_config/service_config_parser.cc +16 -28
- data/src/core/lib/service_config/service_config_parser.h +18 -19
- data/src/core/lib/slice/b64.cc +26 -26
- data/src/core/lib/slice/b64.h +34 -32
- data/src/core/lib/slice/percent_encoding.cc +24 -30
- data/src/core/lib/slice/percent_encoding.h +28 -34
- data/src/core/lib/slice/slice.cc +59 -46
- data/src/core/lib/slice/slice.h +64 -20
- data/src/core/lib/slice/slice_buffer.cc +136 -76
- data/src/core/lib/slice/slice_buffer.h +168 -0
- data/src/core/lib/slice/slice_internal.h +38 -45
- data/src/core/lib/slice/slice_refcount.cc +3 -18
- data/src/core/lib/slice/slice_refcount.h +53 -19
- data/src/core/lib/slice/slice_string_helpers.cc +17 -33
- data/src/core/lib/slice/slice_string_helpers.h +23 -30
- data/src/core/lib/surface/api_trace.cc +17 -17
- data/src/core/lib/surface/api_trace.h +25 -25
- data/src/core/lib/surface/builtins.cc +7 -2
- data/src/core/lib/surface/builtins.h +3 -3
- data/src/core/lib/surface/byte_buffer.cc +26 -21
- data/src/core/lib/surface/byte_buffer_reader.cc +24 -24
- data/src/core/lib/surface/call.cc +2232 -483
- data/src/core/lib/surface/call.h +135 -42
- data/src/core/lib/surface/call_details.cc +22 -23
- data/src/core/lib/surface/call_log_batch.cc +25 -18
- data/src/core/lib/surface/call_test_only.h +34 -31
- data/src/core/lib/surface/call_trace.cc +123 -0
- data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_event_engine.cc → lib/surface/call_trace.h} +12 -10
- data/src/core/lib/surface/channel.cc +217 -284
- data/src/core/lib/surface/channel.h +137 -90
- data/src/core/lib/surface/channel_init.cc +17 -17
- data/src/core/lib/surface/channel_init.h +22 -20
- data/src/core/lib/surface/channel_ping.cc +26 -20
- data/src/core/lib/surface/channel_stack_type.cc +21 -19
- data/src/core/lib/surface/channel_stack_type.h +22 -22
- data/src/core/lib/surface/completion_queue.cc +200 -222
- data/src/core/lib/surface/completion_queue.h +42 -40
- data/src/core/lib/surface/completion_queue_factory.cc +34 -28
- data/src/core/lib/surface/completion_queue_factory.h +21 -23
- data/src/core/lib/surface/event_string.cc +19 -24
- data/src/core/lib/surface/event_string.h +21 -21
- data/src/core/lib/surface/init.cc +57 -114
- data/src/core/lib/surface/init.h +20 -28
- data/src/core/lib/surface/init_internally.cc +25 -0
- data/src/core/lib/surface/init_internally.h +37 -0
- data/src/core/lib/surface/lame_client.cc +87 -130
- data/src/core/lib/surface/lame_client.h +62 -24
- data/src/core/lib/surface/metadata_array.cc +18 -17
- data/src/core/lib/surface/server.cc +406 -174
- data/src/core/lib/surface/server.h +56 -24
- data/src/core/lib/surface/validate_metadata.cc +64 -77
- data/src/core/lib/surface/validate_metadata.h +33 -21
- data/src/core/lib/surface/version.cc +21 -21
- data/src/core/lib/transport/bdp_estimator.cc +19 -19
- data/src/core/lib/transport/bdp_estimator.h +22 -24
- data/src/core/lib/transport/connectivity_state.cc +24 -23
- data/src/core/lib/transport/connectivity_state.h +24 -25
- data/src/core/lib/transport/error_utils.cc +57 -79
- data/src/core/lib/transport/error_utils.h +29 -25
- data/src/core/lib/{channel → transport}/handshaker.cc +75 -58
- data/src/core/lib/{channel → transport}/handshaker.h +59 -48
- data/src/core/lib/transport/handshaker_factory.h +74 -0
- data/src/core/lib/transport/handshaker_registry.cc +61 -0
- data/src/core/lib/transport/handshaker_registry.h +69 -0
- data/src/core/lib/transport/http2_errors.h +22 -22
- data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.cc +78 -65
- data/src/core/lib/transport/http_connect_handshaker.h +42 -0
- data/src/core/lib/transport/metadata_batch.cc +305 -0
- data/src/core/lib/transport/metadata_batch.h +405 -304
- data/src/core/lib/transport/parsed_metadata.cc +2 -4
- data/src/core/lib/transport/parsed_metadata.h +33 -14
- data/src/core/lib/transport/pid_controller.cc +20 -20
- data/src/core/lib/transport/pid_controller.h +27 -27
- data/src/core/lib/transport/status_conversion.cc +23 -23
- data/src/core/lib/transport/status_conversion.h +24 -24
- data/src/core/lib/transport/tcp_connect_handshaker.cc +245 -0
- data/src/core/lib/transport/tcp_connect_handshaker.h +39 -0
- data/src/core/lib/transport/timeout_encoding.cc +24 -28
- data/src/core/lib/transport/timeout_encoding.h +25 -21
- data/src/core/lib/transport/transport.cc +97 -57
- data/src/core/lib/transport/transport.h +243 -213
- data/src/core/lib/transport/transport_fwd.h +20 -0
- data/src/core/lib/transport/transport_impl.h +55 -45
- data/src/core/lib/transport/transport_op_string.cc +40 -39
- data/src/core/lib/uri/uri_parser.cc +12 -4
- data/src/core/lib/uri/uri_parser.h +3 -5
- data/src/core/plugin_registry/grpc_plugin_registry.cc +60 -69
- data/src/core/plugin_registry/grpc_plugin_registry_extra.cc +20 -39
- data/src/core/tsi/alts/crypt/aes_gcm.cc +20 -20
- data/src/core/tsi/alts/crypt/gsec.cc +26 -26
- data/src/core/tsi/alts/crypt/gsec.h +336 -336
- data/src/core/tsi/alts/frame_protector/alts_counter.cc +23 -23
- data/src/core/tsi/alts/frame_protector/alts_counter.h +68 -68
- data/src/core/tsi/alts/frame_protector/alts_crypter.cc +19 -19
- data/src/core/tsi/alts/frame_protector/alts_crypter.h +209 -209
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +72 -71
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.h +40 -40
- data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc +18 -18
- data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h +82 -83
- data/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc +22 -22
- data/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc +22 -22
- data/src/core/tsi/alts/frame_protector/frame_handler.cc +26 -25
- data/src/core/tsi/alts/frame_protector/frame_handler.h +169 -169
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +117 -97
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +104 -104
- data/src/core/tsi/alts/handshaker/alts_shared_resource.cc +18 -17
- data/src/core/tsi/alts/handshaker/alts_shared_resource.h +43 -44
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +51 -43
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +60 -60
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h +24 -24
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +21 -20
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.h +38 -38
- data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +19 -19
- data/src/core/tsi/alts/handshaker/transport_security_common_api.h +105 -104
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +45 -44
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h +40 -41
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +38 -36
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h +35 -36
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol.h +67 -68
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +32 -31
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.h +55 -56
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc +54 -53
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h +141 -142
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +67 -58
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h +43 -44
- data/src/core/tsi/fake_transport_security.cc +146 -110
- data/src/core/tsi/fake_transport_security.h +36 -30
- data/src/core/tsi/local_transport_security.cc +43 -38
- data/src/core/tsi/local_transport_security.h +33 -33
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.cc +9 -4
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.h +3 -3
- data/src/core/tsi/ssl/session_cache/ssl_session.h +23 -21
- data/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc +21 -19
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +18 -17
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.h +31 -22
- data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +27 -20
- data/src/core/tsi/ssl_transport_security.cc +299 -370
- data/src/core/tsi/ssl_transport_security.h +206 -203
- data/src/core/tsi/ssl_transport_security_utils.cc +250 -0
- data/src/core/tsi/ssl_transport_security_utils.h +147 -0
- data/src/core/tsi/ssl_types.h +27 -27
- data/src/core/tsi/transport_security.cc +44 -32
- data/src/core/tsi/transport_security.h +49 -48
- data/src/core/tsi/transport_security_grpc.cc +23 -22
- data/src/core/tsi/transport_security_grpc.h +44 -41
- data/src/core/tsi/transport_security_interface.h +344 -332
- data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.clang +2 -0
- data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.gcc +7 -0
- data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.clang +2 -0
- data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.gcc +7 -0
- data/src/ruby/ext/grpc/ext-export.gcc +1 -1
- data/src/ruby/ext/grpc/extconf.rb +98 -20
- data/src/ruby/ext/grpc/rb_call.c +1 -0
- data/src/ruby/ext/grpc/rb_channel.c +1 -0
- data/src/ruby/ext/grpc/rb_channel_args.c +1 -0
- data/src/ruby/ext/grpc/rb_compression_options.c +1 -1
- data/src/ruby/ext/grpc/rb_grpc.c +1 -0
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.c +38 -38
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +60 -60
- data/src/ruby/ext/grpc/rb_loader.c +6 -2
- data/src/ruby/lib/grpc/errors.rb +1 -1
- data/src/ruby/lib/grpc/generic/bidi_call.rb +2 -0
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/src/ruby/pb/generate_proto_ruby.sh +0 -6
- data/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +4 -0
- data/src/ruby/spec/channel_spec.rb +5 -43
- data/src/ruby/spec/client_server_spec.rb +20 -8
- data/src/ruby/spec/generic/active_call_spec.rb +12 -3
- data/src/ruby/spec/generic/client_stub_spec.rb +23 -23
- data/src/ruby/spec/generic/server_interceptors_spec.rb +1 -1
- data/src/ruby/spec/user_agent_spec.rb +1 -1
- data/third_party/abseil-cpp/absl/algorithm/container.h +57 -58
- data/third_party/abseil-cpp/absl/base/attributes.h +88 -41
- data/third_party/abseil-cpp/absl/base/casts.h +61 -68
- data/third_party/abseil-cpp/absl/base/config.h +221 -39
- data/third_party/abseil-cpp/absl/base/internal/cycleclock.cc +12 -42
- data/third_party/abseil-cpp/absl/base/internal/cycleclock.h +52 -2
- data/third_party/abseil-cpp/absl/base/internal/cycleclock_config.h +55 -0
- data/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +5 -4
- data/third_party/abseil-cpp/absl/base/internal/endian.h +17 -62
- data/third_party/abseil-cpp/absl/base/internal/fast_type_id.h +2 -0
- data/third_party/abseil-cpp/absl/base/internal/inline_variable.h +3 -3
- data/third_party/abseil-cpp/absl/base/internal/invoke.h +54 -0
- data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +2 -2
- data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.h +1 -1
- data/third_party/abseil-cpp/absl/base/internal/prefetch.h +138 -0
- data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc +39 -28
- data/third_party/abseil-cpp/absl/base/internal/raw_logging.h +36 -36
- data/third_party/abseil-cpp/absl/base/internal/spinlock.cc +6 -3
- data/third_party/abseil-cpp/absl/base/internal/spinlock.h +10 -6
- data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc +2 -5
- data/third_party/abseil-cpp/absl/base/internal/spinlock_win32.inc +4 -1
- data/third_party/abseil-cpp/absl/base/internal/strerror.cc +88 -0
- data/third_party/abseil-cpp/absl/base/internal/strerror.h +39 -0
- data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc +14 -11
- data/third_party/abseil-cpp/absl/base/internal/thread_annotations.h +9 -0
- data/third_party/abseil-cpp/absl/base/internal/thread_identity.cc +2 -1
- data/third_party/abseil-cpp/absl/base/internal/thread_identity.h +4 -0
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc +6 -7
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.h +12 -40
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock_config.h +62 -0
- data/third_party/abseil-cpp/absl/base/log_severity.cc +28 -0
- data/third_party/abseil-cpp/absl/base/log_severity.h +51 -0
- data/third_party/abseil-cpp/absl/base/macros.h +4 -21
- data/third_party/abseil-cpp/absl/base/optimization.h +76 -16
- data/third_party/abseil-cpp/absl/base/options.h +1 -7
- data/third_party/abseil-cpp/absl/base/policy_checks.h +15 -13
- data/third_party/abseil-cpp/absl/base/thread_annotations.h +2 -2
- data/third_party/abseil-cpp/absl/cleanup/cleanup.h +140 -0
- data/third_party/abseil-cpp/absl/cleanup/internal/cleanup.h +100 -0
- data/third_party/abseil-cpp/absl/container/fixed_array.h +9 -5
- data/third_party/abseil-cpp/absl/container/flat_hash_map.h +11 -4
- data/third_party/abseil-cpp/absl/container/flat_hash_set.h +503 -0
- data/third_party/abseil-cpp/absl/container/inlined_vector.h +84 -25
- data/third_party/abseil-cpp/absl/container/internal/common.h +9 -8
- data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h +132 -0
- data/third_party/abseil-cpp/absl/container/internal/container_memory.h +23 -29
- data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h +4 -55
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc +116 -23
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h +40 -54
- data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h +165 -66
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc +159 -4
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h +1121 -470
- data/third_party/abseil-cpp/absl/crc/crc32c.cc +99 -0
- data/third_party/abseil-cpp/absl/crc/crc32c.h +183 -0
- data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc +256 -0
- data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.h +57 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc.cc +468 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc.h +91 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +269 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32c.h +39 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32c_inline.h +72 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc +130 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.h +159 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_internal.h +179 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy.h +119 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +75 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc +434 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc +93 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +725 -0
- data/third_party/abseil-cpp/absl/crc/internal/non_temporal_arm_intrinsics.h +79 -0
- data/third_party/abseil-cpp/absl/crc/internal/non_temporal_memcpy.h +180 -0
- data/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc +45 -88
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.cc +67 -38
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.h +1 -1
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc +16 -13
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h +3 -2
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +19 -12
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_arm-inl.inc +8 -3
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_config.h +2 -1
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_generic-inl.inc +1 -1
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc +8 -3
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_riscv-inl.inc +59 -102
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_win32-inl.inc +5 -4
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_x86-inl.inc +41 -11
- data/third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc +18 -4
- data/third_party/abseil-cpp/absl/debugging/symbolize.cc +6 -1
- data/third_party/abseil-cpp/absl/debugging/symbolize_darwin.inc +3 -2
- data/third_party/abseil-cpp/absl/debugging/symbolize_elf.inc +164 -101
- data/third_party/abseil-cpp/absl/debugging/symbolize_win32.inc +7 -6
- data/third_party/abseil-cpp/absl/functional/any_invocable.h +316 -0
- data/third_party/abseil-cpp/absl/functional/bind_front.h +10 -1
- data/third_party/abseil-cpp/absl/functional/function_ref.h +2 -1
- data/third_party/abseil-cpp/absl/functional/internal/any_invocable.h +878 -0
- data/third_party/abseil-cpp/absl/hash/hash.h +82 -8
- data/third_party/abseil-cpp/absl/hash/internal/city.cc +10 -10
- data/third_party/abseil-cpp/absl/hash/internal/hash.h +235 -26
- data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc +3 -14
- data/third_party/abseil-cpp/absl/memory/memory.h +26 -447
- data/third_party/abseil-cpp/absl/meta/type_traits.h +104 -12
- data/third_party/abseil-cpp/absl/numeric/bits.h +1 -1
- data/third_party/abseil-cpp/absl/numeric/int128.cc +14 -10
- data/third_party/abseil-cpp/absl/numeric/int128.h +2 -2
- data/third_party/abseil-cpp/absl/profiling/internal/sample_recorder.h +33 -10
- data/third_party/abseil-cpp/absl/random/bernoulli_distribution.h +4 -4
- data/third_party/abseil-cpp/absl/random/distributions.h +3 -3
- data/third_party/abseil-cpp/absl/random/internal/distribution_caller.h +3 -0
- data/third_party/abseil-cpp/absl/random/internal/fast_uniform_bits.h +4 -2
- data/third_party/abseil-cpp/absl/random/internal/generate_real.h +2 -2
- data/third_party/abseil-cpp/absl/random/internal/nonsecure_base.h +59 -48
- data/third_party/abseil-cpp/absl/random/internal/pcg_engine.h +3 -24
- data/third_party/abseil-cpp/absl/random/internal/pool_urbg.cc +9 -9
- data/third_party/abseil-cpp/absl/random/internal/randen.h +5 -11
- data/third_party/abseil-cpp/absl/random/internal/randen_detect.cc +6 -2
- data/third_party/abseil-cpp/absl/random/internal/randen_engine.h +48 -23
- data/third_party/abseil-cpp/absl/random/internal/salted_seed_seq.h +24 -26
- data/third_party/abseil-cpp/absl/random/internal/seed_material.cc +2 -2
- data/third_party/abseil-cpp/absl/random/internal/traits.h +53 -5
- data/third_party/abseil-cpp/absl/random/internal/uniform_helper.h +5 -5
- data/third_party/abseil-cpp/absl/random/internal/wide_multiply.h +33 -48
- data/third_party/abseil-cpp/absl/random/log_uniform_int_distribution.h +9 -10
- data/third_party/abseil-cpp/absl/random/poisson_distribution.h +7 -4
- data/third_party/abseil-cpp/absl/random/random.h +6 -6
- data/third_party/abseil-cpp/absl/random/seed_sequences.h +1 -0
- data/third_party/abseil-cpp/absl/random/uniform_int_distribution.h +2 -2
- data/third_party/abseil-cpp/absl/random/uniform_real_distribution.h +1 -1
- data/third_party/abseil-cpp/absl/random/zipf_distribution.h +4 -3
- data/third_party/abseil-cpp/absl/status/internal/status_internal.h +18 -0
- data/third_party/abseil-cpp/absl/status/status.cc +193 -14
- data/third_party/abseil-cpp/absl/status/status.h +24 -14
- data/third_party/abseil-cpp/absl/status/statusor.h +9 -3
- data/third_party/abseil-cpp/absl/strings/ascii.cc +5 -5
- data/third_party/abseil-cpp/absl/strings/ascii.h +4 -4
- data/third_party/abseil-cpp/absl/strings/charconv.cc +534 -96
- data/third_party/abseil-cpp/absl/strings/cord.cc +257 -924
- data/third_party/abseil-cpp/absl/strings/cord.h +268 -156
- data/third_party/abseil-cpp/absl/strings/cord_analysis.cc +188 -0
- data/third_party/abseil-cpp/absl/strings/cord_analysis.h +44 -0
- data/third_party/abseil-cpp/absl/strings/cord_buffer.cc +30 -0
- data/third_party/abseil-cpp/absl/strings/cord_buffer.h +575 -0
- data/third_party/abseil-cpp/absl/strings/escaping.cc +73 -62
- data/third_party/abseil-cpp/absl/strings/escaping.h +24 -19
- data/third_party/abseil-cpp/absl/strings/internal/char_map.h +14 -12
- data/third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc +4 -4
- data/third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc +2 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_data_edge.h +63 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.cc +20 -32
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +441 -150
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc +157 -53
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h +70 -73
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc +8 -6
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.h +4 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc +3 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.h +5 -4
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc +7 -74
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc +56 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.h +103 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_flat.h +58 -17
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_ring.cc +13 -11
- data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.h +7 -15
- data/third_party/abseil-cpp/absl/strings/internal/cordz_handle.h +3 -3
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.cc +19 -43
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.h +7 -7
- data/third_party/abseil-cpp/absl/strings/internal/cordz_statistics.h +5 -4
- data/third_party/abseil-cpp/absl/strings/internal/cordz_update_tracker.h +4 -2
- data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc +93 -0
- data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.h +34 -0
- data/third_party/abseil-cpp/absl/strings/internal/escaping.cc +18 -15
- data/third_party/abseil-cpp/absl/strings/internal/escaping.h +7 -9
- data/third_party/abseil-cpp/absl/strings/internal/has_absl_stringify.h +55 -0
- data/third_party/abseil-cpp/absl/strings/internal/memutil.cc +9 -6
- data/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +14 -7
- data/third_party/abseil-cpp/absl/strings/internal/ostringstream.h +35 -10
- data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc +113 -46
- data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.h +127 -30
- data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc +3 -2
- data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.h +42 -10
- data/third_party/abseil-cpp/absl/strings/internal/str_format/checker.h +56 -289
- data/third_party/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h +351 -0
- data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc +6 -6
- data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.h +9 -4
- data/third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc +215 -181
- data/third_party/abseil-cpp/absl/strings/internal/str_format/output.h +2 -1
- data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc +10 -209
- data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.h +14 -103
- data/third_party/abseil-cpp/absl/strings/internal/str_join_internal.h +9 -6
- data/third_party/abseil-cpp/absl/strings/internal/str_split_internal.h +2 -1
- data/third_party/abseil-cpp/absl/strings/internal/string_constant.h +10 -2
- data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc +28 -0
- data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.h +57 -0
- data/third_party/abseil-cpp/absl/strings/internal/utf8.cc +9 -9
- data/third_party/abseil-cpp/absl/strings/numbers.cc +42 -39
- data/third_party/abseil-cpp/absl/strings/numbers.h +26 -23
- data/third_party/abseil-cpp/absl/strings/str_cat.cc +9 -6
- data/third_party/abseil-cpp/absl/strings/str_cat.h +70 -16
- data/third_party/abseil-cpp/absl/strings/str_format.h +71 -9
- data/third_party/abseil-cpp/absl/strings/str_join.h +9 -15
- data/third_party/abseil-cpp/absl/strings/str_split.h +1 -2
- data/third_party/abseil-cpp/absl/strings/string_view.cc +8 -19
- data/third_party/abseil-cpp/absl/strings/string_view.h +6 -12
- data/third_party/abseil-cpp/absl/strings/strip.h +8 -6
- data/third_party/abseil-cpp/absl/strings/substitute.cc +8 -6
- data/third_party/abseil-cpp/absl/strings/substitute.h +55 -21
- data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc +9 -6
- data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.h +0 -4
- data/third_party/abseil-cpp/absl/synchronization/internal/futex.h +20 -17
- data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc +37 -31
- data/third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.h +22 -8
- data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc +0 -4
- data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.h +1 -6
- data/third_party/abseil-cpp/absl/synchronization/internal/waiter.cc +0 -25
- data/third_party/abseil-cpp/absl/synchronization/internal/waiter.h +10 -4
- data/third_party/abseil-cpp/absl/synchronization/mutex.cc +172 -88
- data/third_party/abseil-cpp/absl/synchronization/mutex.h +102 -55
- data/third_party/abseil-cpp/absl/synchronization/notification.cc +0 -1
- data/third_party/abseil-cpp/absl/synchronization/notification.h +3 -3
- data/third_party/abseil-cpp/absl/time/civil_time.cc +26 -0
- data/third_party/abseil-cpp/absl/time/civil_time.h +25 -0
- data/third_party/abseil-cpp/absl/time/clock.cc +17 -11
- data/third_party/abseil-cpp/absl/time/duration.cc +12 -11
- data/third_party/abseil-cpp/absl/time/format.cc +2 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h +11 -7
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc +1 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc +26 -5
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc +7 -6
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc +36 -35
- data/third_party/abseil-cpp/absl/time/time.cc +2 -2
- data/third_party/abseil-cpp/absl/time/time.h +268 -169
- data/third_party/abseil-cpp/absl/types/internal/optional.h +8 -0
- data/third_party/abseil-cpp/absl/types/internal/span.h +30 -19
- data/third_party/abseil-cpp/absl/types/internal/variant.h +28 -40
- data/third_party/abseil-cpp/absl/types/optional.h +17 -14
- data/third_party/abseil-cpp/absl/types/span.h +31 -8
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_genn.c +1 -1
- data/third_party/boringssl-with-bazel/src/include/openssl/x509v3.h +1 -2
- data/third_party/re2/re2/bitstate.cc +3 -3
- data/third_party/re2/re2/dfa.cc +13 -13
- data/third_party/re2/re2/nfa.cc +4 -4
- data/third_party/re2/re2/onepass.cc +2 -2
- data/third_party/re2/re2/prefilter_tree.cc +27 -59
- data/third_party/re2/re2/prefilter_tree.h +3 -2
- data/third_party/re2/re2/prog.cc +11 -2
- data/third_party/re2/re2/prog.h +17 -5
- data/third_party/re2/re2/re2.cc +6 -11
- data/third_party/re2/re2/re2.h +1 -1
- data/third_party/re2/re2/regexp.cc +1 -2
- data/third_party/re2/re2/stringpiece.h +10 -7
- data/third_party/re2/re2/unicode_casefold.cc +25 -11
- data/third_party/re2/re2/unicode_groups.cc +319 -151
- data/third_party/re2/re2/walker-inl.h +3 -2
- data/third_party/re2/util/mutex.h +4 -4
- data/third_party/upb/third_party/utf8_range/utf8_range.h +1 -1
- data/third_party/upb/upb/arena.c +277 -0
- data/third_party/upb/upb/arena.h +225 -0
- data/third_party/upb/upb/array.c +114 -0
- data/third_party/upb/upb/array.h +83 -0
- data/third_party/upb/upb/collections.h +36 -0
- data/third_party/upb/upb/decode.c +161 -65
- data/third_party/upb/upb/decode.h +1 -0
- data/third_party/upb/upb/decode_fast.c +1 -1
- data/third_party/upb/upb/def.c +10 -2
- data/third_party/upb/upb/def.h +8 -1
- data/third_party/upb/upb/def.hpp +7 -4
- data/third_party/upb/upb/encode.c +29 -20
- data/third_party/upb/upb/encode.h +16 -6
- data/third_party/upb/upb/extension_registry.c +93 -0
- data/third_party/upb/upb/extension_registry.h +84 -0
- data/third_party/upb/upb/{decode_internal.h → internal/decode.h} +5 -5
- data/third_party/upb/upb/internal/table.h +385 -0
- data/third_party/upb/upb/{upb_internal.h → internal/upb.h} +3 -3
- data/third_party/upb/upb/internal/vsnprintf_compat.h +52 -0
- data/third_party/upb/upb/json_decode.c +1512 -0
- data/third_party/upb/upb/json_decode.h +47 -0
- data/third_party/upb/upb/json_encode.c +7 -3
- data/third_party/upb/upb/json_encode.h +6 -3
- data/third_party/upb/upb/map.c +108 -0
- data/third_party/upb/upb/map.h +117 -0
- data/third_party/upb/upb/message_value.h +66 -0
- data/third_party/upb/upb/mini_table.c +1147 -0
- data/third_party/upb/upb/mini_table.h +189 -0
- data/third_party/upb/upb/mini_table.hpp +112 -0
- data/third_party/upb/upb/msg.c +2 -62
- data/third_party/upb/upb/msg.h +2 -45
- data/third_party/upb/upb/msg_internal.h +28 -22
- data/third_party/upb/upb/port_def.inc +2 -1
- data/third_party/upb/upb/port_undef.inc +1 -0
- data/third_party/upb/upb/reflection.c +2 -159
- data/third_party/upb/upb/reflection.h +2 -112
- data/third_party/upb/upb/status.c +86 -0
- data/third_party/upb/upb/status.h +66 -0
- data/third_party/upb/upb/table.c +2 -2
- data/third_party/upb/upb/table_internal.h +3 -352
- data/third_party/upb/upb/text_encode.c +3 -2
- data/third_party/upb/upb/upb.c +4 -290
- data/third_party/upb/upb/upb.h +7 -196
- data/third_party/zlib/compress.c +3 -3
- data/third_party/zlib/crc32.c +21 -12
- data/third_party/zlib/deflate.c +112 -106
- data/third_party/zlib/deflate.h +2 -2
- data/third_party/zlib/gzlib.c +1 -1
- data/third_party/zlib/gzread.c +3 -5
- data/third_party/zlib/gzwrite.c +1 -1
- data/third_party/zlib/infback.c +10 -7
- data/third_party/zlib/inflate.c +5 -2
- data/third_party/zlib/inftrees.c +2 -2
- data/third_party/zlib/inftrees.h +1 -1
- data/third_party/zlib/trees.c +61 -62
- data/third_party/zlib/uncompr.c +2 -2
- data/third_party/zlib/zconf.h +16 -3
- data/third_party/zlib/zlib.h +10 -10
- data/third_party/zlib/zutil.c +9 -7
- data/third_party/zlib/zutil.h +1 -0
- metadata +372 -102
- data/include/grpc/impl/codegen/gpr_slice.h +0 -71
- data/src/core/ext/filters/client_channel/http_connect_handshaker.h +0 -42
- data/src/core/ext/filters/client_channel/lb_policy_factory.h +0 -48
- data/src/core/ext/filters/client_channel/lb_policy_registry.cc +0 -185
- data/src/core/ext/filters/client_channel/lb_policy_registry.h +0 -65
- data/src/core/ext/filters/client_channel/proxy_mapper.h +0 -54
- data/src/core/ext/filters/client_channel/proxy_mapper_registry.cc +0 -89
- data/src/core/ext/filters/client_channel/proxy_mapper_registry.h +0 -50
- data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +0 -186
- data/src/core/ext/filters/fault_injection/service_config_parser.cc +0 -179
- data/src/core/ext/filters/http/message_compress/message_compress_filter.cc +0 -456
- data/src/core/ext/filters/http/message_compress/message_compress_filter.h +0 -53
- data/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +0 -386
- data/src/core/ext/filters/server_config_selector/server_config_selector.cc +0 -61
- data/src/core/ext/xds/certificate_provider_registry.cc +0 -103
- data/src/core/ext/xds/certificate_provider_registry.h +0 -57
- data/src/core/lib/channel/handshaker_factory.h +0 -50
- data/src/core/lib/channel/handshaker_registry.cc +0 -50
- data/src/core/lib/channel/handshaker_registry.h +0 -71
- data/src/core/lib/event_engine/sockaddr.cc +0 -40
- data/src/core/lib/event_engine/sockaddr.h +0 -44
- data/src/core/lib/gpr/env.h +0 -40
- data/src/core/lib/gpr/env_linux.cc +0 -75
- data/src/core/lib/gpr/env_posix.cc +0 -46
- data/src/core/lib/gpr/env_windows.cc +0 -74
- data/src/core/lib/gpr/murmur_hash.cc +0 -82
- data/src/core/lib/gpr/murmur_hash.h +0 -29
- data/src/core/lib/gpr/string_windows.h +0 -32
- data/src/core/lib/gpr/tls.h +0 -158
- data/src/core/lib/gprpp/capture.h +0 -76
- data/src/core/lib/iomgr/error_internal.h +0 -66
- data/src/core/lib/iomgr/event_engine/endpoint.cc +0 -172
- data/src/core/lib/iomgr/event_engine/endpoint.h +0 -52
- data/src/core/lib/iomgr/event_engine/iomgr.cc +0 -85
- data/src/core/lib/iomgr/event_engine/pollset.cc +0 -87
- data/src/core/lib/iomgr/event_engine/resolved_address_internal.cc +0 -47
- data/src/core/lib/iomgr/event_engine/resolver.cc +0 -133
- data/src/core/lib/iomgr/event_engine/resolver.h +0 -56
- data/src/core/lib/iomgr/event_engine/tcp.cc +0 -296
- data/src/core/lib/iomgr/event_engine/timer.cc +0 -62
- data/src/core/lib/iomgr/executor/mpmcqueue.cc +0 -182
- data/src/core/lib/iomgr/executor/mpmcqueue.h +0 -171
- data/src/core/lib/iomgr/executor/threadpool.cc +0 -136
- data/src/core/lib/iomgr/executor/threadpool.h +0 -150
- data/src/core/lib/iomgr/time_averaged_stats.cc +0 -64
- data/src/core/lib/iomgr/time_averaged_stats.h +0 -72
- data/src/core/lib/profiling/basic_timers.cc +0 -295
- data/src/core/lib/profiling/stap_timers.cc +0 -50
- data/src/core/lib/profiling/timers.h +0 -94
- data/src/core/lib/promise/call_push_pull.h +0 -144
- data/src/core/lib/security/security_connector/load_system_roots_linux.h +0 -46
- data/src/core/lib/slice/slice_api.cc +0 -39
- data/src/core/lib/slice/slice_refcount_base.h +0 -61
- data/src/core/lib/slice/slice_split.cc +0 -100
- data/src/core/lib/slice/slice_split.h +0 -40
- data/src/core/lib/transport/byte_stream.cc +0 -164
- data/src/core/lib/transport/byte_stream.h +0 -166
- data/third_party/abseil-cpp/absl/container/internal/have_sse.h +0 -50
@@ -1,60 +1,95 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
1
|
+
//
|
2
|
+
//
|
3
|
+
// Copyright 2015 gRPC authors.
|
4
|
+
//
|
5
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
// you may not use this file except in compliance with the License.
|
7
|
+
// You may obtain a copy of the License at
|
8
|
+
//
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
//
|
11
|
+
// Unless required by applicable law or agreed to in writing, software
|
12
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
// See the License for the specific language governing permissions and
|
15
|
+
// limitations under the License.
|
16
|
+
//
|
17
|
+
//
|
18
18
|
|
19
19
|
#include <grpc/support/port_platform.h>
|
20
20
|
|
21
21
|
#include "src/core/lib/surface/call.h"
|
22
22
|
|
23
|
-
#include <
|
23
|
+
#include <inttypes.h>
|
24
24
|
#include <limits.h>
|
25
|
-
#include <stdio.h>
|
26
25
|
#include <stdlib.h>
|
27
|
-
#include <string.h>
|
28
26
|
|
27
|
+
#include <algorithm>
|
29
28
|
#include <atomic>
|
29
|
+
#include <initializer_list>
|
30
|
+
#include <memory>
|
31
|
+
#include <new>
|
30
32
|
#include <string>
|
31
|
-
|
33
|
+
#include <type_traits>
|
34
|
+
#include <utility>
|
35
|
+
#include <vector>
|
36
|
+
|
37
|
+
#include "absl/base/thread_annotations.h"
|
38
|
+
#include "absl/cleanup/cleanup.h"
|
39
|
+
#include "absl/meta/type_traits.h"
|
40
|
+
#include "absl/status/status.h"
|
32
41
|
#include "absl/strings/str_cat.h"
|
33
42
|
#include "absl/strings/str_format.h"
|
43
|
+
#include "absl/strings/str_join.h"
|
44
|
+
#include "absl/strings/string_view.h"
|
45
|
+
#include "absl/types/variant.h"
|
34
46
|
|
47
|
+
#include <grpc/byte_buffer.h>
|
35
48
|
#include <grpc/compression.h>
|
49
|
+
#include <grpc/event_engine/event_engine.h>
|
36
50
|
#include <grpc/grpc.h>
|
51
|
+
#include <grpc/impl/propagation_bits.h>
|
37
52
|
#include <grpc/slice.h>
|
53
|
+
#include <grpc/slice_buffer.h>
|
54
|
+
#include <grpc/status.h>
|
38
55
|
#include <grpc/support/alloc.h>
|
56
|
+
#include <grpc/support/atm.h>
|
39
57
|
#include <grpc/support/log.h>
|
40
58
|
#include <grpc/support/string_util.h>
|
59
|
+
#include <grpc/support/time.h>
|
41
60
|
|
61
|
+
#include "src/core/lib/channel/call_finalization.h"
|
42
62
|
#include "src/core/lib/channel/channel_stack.h"
|
63
|
+
#include "src/core/lib/channel/channelz.h"
|
43
64
|
#include "src/core/lib/channel/context.h"
|
65
|
+
#include "src/core/lib/channel/status_util.h"
|
44
66
|
#include "src/core/lib/compression/compression_internal.h"
|
45
67
|
#include "src/core/lib/debug/stats.h"
|
68
|
+
#include "src/core/lib/debug/stats_data.h"
|
69
|
+
#include "src/core/lib/experiments/experiments.h"
|
46
70
|
#include "src/core/lib/gpr/alloc.h"
|
47
|
-
#include "src/core/lib/gpr/string.h"
|
48
71
|
#include "src/core/lib/gpr/time_precise.h"
|
49
72
|
#include "src/core/lib/gpr/useful.h"
|
50
|
-
#include "src/core/lib/gprpp/
|
73
|
+
#include "src/core/lib/gprpp/bitset.h"
|
74
|
+
#include "src/core/lib/gprpp/cpp_impl_of.h"
|
75
|
+
#include "src/core/lib/gprpp/crash.h"
|
76
|
+
#include "src/core/lib/gprpp/debug_location.h"
|
51
77
|
#include "src/core/lib/gprpp/ref_counted.h"
|
52
|
-
#include "src/core/lib/
|
53
|
-
#include "src/core/lib/
|
78
|
+
#include "src/core/lib/gprpp/ref_counted_ptr.h"
|
79
|
+
#include "src/core/lib/gprpp/status_helper.h"
|
80
|
+
#include "src/core/lib/gprpp/sync.h"
|
81
|
+
#include "src/core/lib/iomgr/call_combiner.h"
|
82
|
+
#include "src/core/lib/iomgr/exec_ctx.h"
|
83
|
+
#include "src/core/lib/iomgr/polling_entity.h"
|
84
|
+
#include "src/core/lib/promise/activity.h"
|
85
|
+
#include "src/core/lib/promise/arena_promise.h"
|
86
|
+
#include "src/core/lib/promise/context.h"
|
87
|
+
#include "src/core/lib/promise/detail/basic_seq.h"
|
88
|
+
#include "src/core/lib/promise/pipe.h"
|
89
|
+
#include "src/core/lib/promise/poll.h"
|
54
90
|
#include "src/core/lib/resource_quota/arena.h"
|
91
|
+
#include "src/core/lib/slice/slice_buffer.h"
|
55
92
|
#include "src/core/lib/slice/slice_internal.h"
|
56
|
-
#include "src/core/lib/slice/slice_split.h"
|
57
|
-
#include "src/core/lib/slice/slice_string_helpers.h"
|
58
93
|
#include "src/core/lib/surface/api_trace.h"
|
59
94
|
#include "src/core/lib/surface/call_test_only.h"
|
60
95
|
#include "src/core/lib/surface/channel.h"
|
@@ -62,13 +97,19 @@
|
|
62
97
|
#include "src/core/lib/surface/server.h"
|
63
98
|
#include "src/core/lib/surface/validate_metadata.h"
|
64
99
|
#include "src/core/lib/transport/error_utils.h"
|
100
|
+
#include "src/core/lib/transport/metadata_batch.h"
|
65
101
|
#include "src/core/lib/transport/transport.h"
|
66
102
|
|
67
103
|
grpc_core::TraceFlag grpc_call_error_trace(false, "call_error");
|
68
104
|
grpc_core::TraceFlag grpc_compression_trace(false, "compression");
|
105
|
+
grpc_core::TraceFlag grpc_call_trace(false, "call");
|
106
|
+
grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount");
|
69
107
|
|
70
108
|
namespace grpc_core {
|
71
109
|
|
110
|
+
///////////////////////////////////////////////////////////////////////////////
|
111
|
+
// Call
|
112
|
+
|
72
113
|
class Call : public CppImplOf<Call, grpc_call> {
|
73
114
|
public:
|
74
115
|
Arena* arena() { return arena_; }
|
@@ -81,12 +122,13 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
81
122
|
void CancelWithStatus(grpc_status_code status, const char* description);
|
82
123
|
virtual void CancelWithError(grpc_error_handle error) = 0;
|
83
124
|
virtual void SetCompletionQueue(grpc_completion_queue* cq) = 0;
|
84
|
-
|
125
|
+
char* GetPeer();
|
85
126
|
virtual grpc_call_error StartBatch(const grpc_op* ops, size_t nops,
|
86
127
|
void* notify_tag,
|
87
128
|
bool is_notify_tag_closure) = 0;
|
88
129
|
virtual bool failed_before_recv_message() const = 0;
|
89
130
|
virtual bool is_trailers_only() const = 0;
|
131
|
+
virtual absl::string_view GetServerAuthority() const = 0;
|
90
132
|
virtual void ExternalRef() = 0;
|
91
133
|
virtual void ExternalUnref() = 0;
|
92
134
|
virtual void InternalRef(const char* reason) = 0;
|
@@ -102,12 +144,19 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
102
144
|
// for that functionality be invented)
|
103
145
|
virtual grpc_call_stack* call_stack() = 0;
|
104
146
|
|
147
|
+
gpr_atm* peer_string_atm_ptr() { return &peer_string_; }
|
148
|
+
|
105
149
|
protected:
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
150
|
+
// The maximum number of concurrent batches possible.
|
151
|
+
// Based upon the maximum number of individually queueable ops in the batch
|
152
|
+
// api:
|
153
|
+
// - initial metadata send
|
154
|
+
// - message send
|
155
|
+
// - status/close send (depending on client/server)
|
156
|
+
// - initial metadata recv
|
157
|
+
// - message recv
|
158
|
+
// - status/close recv (depending on client/server)
|
159
|
+
static constexpr size_t kMaxConcurrentBatches = 6;
|
111
160
|
|
112
161
|
struct ParentCall {
|
113
162
|
Mutex child_list_mu;
|
@@ -117,15 +166,32 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
117
166
|
struct ChildCall {
|
118
167
|
explicit ChildCall(Call* parent) : parent(parent) {}
|
119
168
|
Call* parent;
|
120
|
-
|
121
|
-
|
122
|
-
|
169
|
+
/// siblings: children of the same parent form a list, and this list is
|
170
|
+
/// protected under
|
171
|
+
/// parent->mu
|
123
172
|
Call* sibling_next = nullptr;
|
124
173
|
Call* sibling_prev = nullptr;
|
125
174
|
};
|
126
175
|
|
176
|
+
Call(Arena* arena, bool is_client, Timestamp send_deadline,
|
177
|
+
RefCountedPtr<Channel> channel)
|
178
|
+
: channel_(std::move(channel)),
|
179
|
+
arena_(arena),
|
180
|
+
send_deadline_(send_deadline),
|
181
|
+
is_client_(is_client) {
|
182
|
+
GPR_DEBUG_ASSERT(arena_ != nullptr);
|
183
|
+
GPR_DEBUG_ASSERT(channel_ != nullptr);
|
184
|
+
}
|
185
|
+
virtual ~Call() = default;
|
186
|
+
|
187
|
+
void DeleteThis();
|
188
|
+
|
127
189
|
ParentCall* GetOrCreateParentCall();
|
128
190
|
ParentCall* parent_call();
|
191
|
+
Channel* channel() {
|
192
|
+
GPR_DEBUG_ASSERT(channel_ != nullptr);
|
193
|
+
return channel_.get();
|
194
|
+
}
|
129
195
|
|
130
196
|
absl::Status InitParent(Call* parent, uint32_t propagation_mask);
|
131
197
|
void PublishToParent(Call* parent);
|
@@ -137,7 +203,10 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
137
203
|
send_deadline_ = send_deadline;
|
138
204
|
}
|
139
205
|
|
206
|
+
void ClearPeerString() { gpr_atm_rel_store(&peer_string_, 0); }
|
207
|
+
|
140
208
|
private:
|
209
|
+
RefCountedPtr<Channel> channel_;
|
141
210
|
Arena* const arena_;
|
142
211
|
std::atomic<ParentCall*> parent_call_{nullptr};
|
143
212
|
ChildCall* child_ = nullptr;
|
@@ -145,11 +214,151 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
145
214
|
const bool is_client_;
|
146
215
|
// flag indicating that cancellation is inherited
|
147
216
|
bool cancellation_is_inherited_ = false;
|
217
|
+
// A char* indicating the peer name.
|
218
|
+
gpr_atm peer_string_ = 0;
|
148
219
|
};
|
149
220
|
|
221
|
+
Call::ParentCall* Call::GetOrCreateParentCall() {
|
222
|
+
ParentCall* p = parent_call_.load(std::memory_order_acquire);
|
223
|
+
if (p == nullptr) {
|
224
|
+
p = arena_->New<ParentCall>();
|
225
|
+
ParentCall* expected = nullptr;
|
226
|
+
if (!parent_call_.compare_exchange_strong(expected, p,
|
227
|
+
std::memory_order_release,
|
228
|
+
std::memory_order_relaxed)) {
|
229
|
+
p->~ParentCall();
|
230
|
+
p = expected;
|
231
|
+
}
|
232
|
+
}
|
233
|
+
return p;
|
234
|
+
}
|
235
|
+
|
236
|
+
Call::ParentCall* Call::parent_call() {
|
237
|
+
return parent_call_.load(std::memory_order_acquire);
|
238
|
+
}
|
239
|
+
|
240
|
+
absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
|
241
|
+
child_ = arena()->New<ChildCall>(parent);
|
242
|
+
|
243
|
+
parent->InternalRef("child");
|
244
|
+
GPR_ASSERT(is_client_);
|
245
|
+
GPR_ASSERT(!parent->is_client_);
|
246
|
+
|
247
|
+
if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
|
248
|
+
send_deadline_ = std::min(send_deadline_, parent->send_deadline_);
|
249
|
+
}
|
250
|
+
// for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
|
251
|
+
// GRPC_PROPAGATE_STATS_CONTEXT
|
252
|
+
// TODO(ctiller): This should change to use the appropriate census start_op
|
253
|
+
// call.
|
254
|
+
if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
|
255
|
+
if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
|
256
|
+
return absl::UnknownError(
|
257
|
+
"Census tracing propagation requested without Census context "
|
258
|
+
"propagation");
|
259
|
+
}
|
260
|
+
ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
|
261
|
+
nullptr);
|
262
|
+
} else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
263
|
+
return absl::UnknownError(
|
264
|
+
"Census context propagation requested without Census tracing "
|
265
|
+
"propagation");
|
266
|
+
}
|
267
|
+
if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
|
268
|
+
cancellation_is_inherited_ = true;
|
269
|
+
}
|
270
|
+
return absl::OkStatus();
|
271
|
+
}
|
272
|
+
|
273
|
+
void Call::PublishToParent(Call* parent) {
|
274
|
+
ChildCall* cc = child_;
|
275
|
+
ParentCall* pc = parent->GetOrCreateParentCall();
|
276
|
+
MutexLock lock(&pc->child_list_mu);
|
277
|
+
if (pc->first_child == nullptr) {
|
278
|
+
pc->first_child = this;
|
279
|
+
cc->sibling_next = cc->sibling_prev = this;
|
280
|
+
} else {
|
281
|
+
cc->sibling_next = pc->first_child;
|
282
|
+
cc->sibling_prev = pc->first_child->child_->sibling_prev;
|
283
|
+
cc->sibling_next->child_->sibling_prev =
|
284
|
+
cc->sibling_prev->child_->sibling_next = this;
|
285
|
+
}
|
286
|
+
if (parent->Completed()) {
|
287
|
+
CancelWithError(absl::CancelledError());
|
288
|
+
}
|
289
|
+
}
|
290
|
+
|
291
|
+
void Call::MaybeUnpublishFromParent() {
|
292
|
+
ChildCall* cc = child_;
|
293
|
+
if (cc == nullptr) return;
|
294
|
+
|
295
|
+
ParentCall* pc = cc->parent->parent_call();
|
296
|
+
{
|
297
|
+
MutexLock lock(&pc->child_list_mu);
|
298
|
+
if (this == pc->first_child) {
|
299
|
+
pc->first_child = cc->sibling_next;
|
300
|
+
if (this == pc->first_child) {
|
301
|
+
pc->first_child = nullptr;
|
302
|
+
}
|
303
|
+
}
|
304
|
+
cc->sibling_prev->child_->sibling_next = cc->sibling_next;
|
305
|
+
cc->sibling_next->child_->sibling_prev = cc->sibling_prev;
|
306
|
+
}
|
307
|
+
cc->parent->InternalUnref("child");
|
308
|
+
}
|
309
|
+
|
310
|
+
void Call::CancelWithStatus(grpc_status_code status, const char* description) {
|
311
|
+
// copying 'description' is needed to ensure the grpc_call_cancel_with_status
|
312
|
+
// guarantee that can be short-lived.
|
313
|
+
CancelWithError(grpc_error_set_int(
|
314
|
+
grpc_error_set_str(GRPC_ERROR_CREATE(description),
|
315
|
+
StatusStrProperty::kGrpcMessage, description),
|
316
|
+
StatusIntProperty::kRpcStatus, status));
|
317
|
+
}
|
318
|
+
|
319
|
+
void Call::PropagateCancellationToChildren() {
|
320
|
+
ParentCall* pc = parent_call();
|
321
|
+
if (pc != nullptr) {
|
322
|
+
Call* child;
|
323
|
+
MutexLock lock(&pc->child_list_mu);
|
324
|
+
child = pc->first_child;
|
325
|
+
if (child != nullptr) {
|
326
|
+
do {
|
327
|
+
Call* next_child_call = child->child_->sibling_next;
|
328
|
+
if (child->cancellation_is_inherited_) {
|
329
|
+
child->InternalRef("propagate_cancel");
|
330
|
+
child->CancelWithError(absl::CancelledError());
|
331
|
+
child->InternalUnref("propagate_cancel");
|
332
|
+
}
|
333
|
+
child = next_child_call;
|
334
|
+
} while (child != pc->first_child);
|
335
|
+
}
|
336
|
+
}
|
337
|
+
}
|
338
|
+
|
339
|
+
char* Call::GetPeer() {
|
340
|
+
char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
|
341
|
+
if (peer_string != nullptr) return gpr_strdup(peer_string);
|
342
|
+
peer_string = grpc_channel_get_target(channel_->c_ptr());
|
343
|
+
if (peer_string != nullptr) return peer_string;
|
344
|
+
return gpr_strdup("unknown");
|
345
|
+
}
|
346
|
+
|
347
|
+
void Call::DeleteThis() {
|
348
|
+
RefCountedPtr<Channel> channel = std::move(channel_);
|
349
|
+
Arena* arena = arena_;
|
350
|
+
this->~Call();
|
351
|
+
channel->UpdateCallSizeEstimate(arena->TotalUsedBytes());
|
352
|
+
arena->Destroy();
|
353
|
+
}
|
354
|
+
|
355
|
+
///////////////////////////////////////////////////////////////////////////////
|
356
|
+
// FilterStackCall
|
357
|
+
// To be removed once promise conversion is complete
|
358
|
+
|
150
359
|
class FilterStackCall final : public Call {
|
151
360
|
public:
|
152
|
-
~FilterStackCall() {
|
361
|
+
~FilterStackCall() override {
|
153
362
|
for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
|
154
363
|
if (context_[i].destroy) {
|
155
364
|
context_[i].destroy(context_[i].value);
|
@@ -184,7 +393,6 @@ class FilterStackCall final : public Call {
|
|
184
393
|
|
185
394
|
void CancelWithError(grpc_error_handle error) override;
|
186
395
|
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
187
|
-
char* GetPeer() override;
|
188
396
|
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
189
397
|
bool is_notify_tag_closure) override;
|
190
398
|
void ExternalRef() override { ext_ref_.Ref(); }
|
@@ -217,6 +425,13 @@ class FilterStackCall final : public Call {
|
|
217
425
|
return call_failed_before_recv_message_;
|
218
426
|
}
|
219
427
|
|
428
|
+
absl::string_view GetServerAuthority() const override {
|
429
|
+
const Slice* authority_metadata =
|
430
|
+
recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
|
431
|
+
if (authority_metadata == nullptr) return "";
|
432
|
+
return authority_metadata->as_string_view();
|
433
|
+
}
|
434
|
+
|
220
435
|
grpc_compression_algorithm test_only_compression_algorithm() override {
|
221
436
|
return incoming_compression_algorithm_;
|
222
437
|
}
|
@@ -235,57 +450,78 @@ class FilterStackCall final : public Call {
|
|
235
450
|
}
|
236
451
|
|
237
452
|
private:
|
238
|
-
// The maximum number of concurrent batches possible.
|
239
|
-
// Based upon the maximum number of individually queueable ops in the batch
|
240
|
-
// api:
|
241
|
-
// - initial metadata send
|
242
|
-
// - message send
|
243
|
-
// - status/close send (depending on client/server)
|
244
|
-
// - initial metadata recv
|
245
|
-
// - message recv
|
246
|
-
// - status/close recv (depending on client/server)
|
247
|
-
static constexpr size_t kMaxConcurrentBatches = 6;
|
248
|
-
|
249
453
|
static constexpr gpr_atm kRecvNone = 0;
|
250
454
|
static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
|
251
455
|
|
456
|
+
enum class PendingOp {
|
457
|
+
kRecvMessage,
|
458
|
+
kRecvInitialMetadata,
|
459
|
+
kRecvTrailingMetadata,
|
460
|
+
kSends
|
461
|
+
};
|
462
|
+
static intptr_t PendingOpMask(PendingOp op) {
|
463
|
+
return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
|
464
|
+
}
|
465
|
+
static std::string PendingOpString(intptr_t pending_ops) {
|
466
|
+
std::vector<absl::string_view> pending_op_strings;
|
467
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
|
468
|
+
pending_op_strings.push_back("kRecvMessage");
|
469
|
+
}
|
470
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
|
471
|
+
pending_op_strings.push_back("kRecvInitialMetadata");
|
472
|
+
}
|
473
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
|
474
|
+
pending_op_strings.push_back("kRecvTrailingMetadata");
|
475
|
+
}
|
476
|
+
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
477
|
+
pending_op_strings.push_back("kSends");
|
478
|
+
}
|
479
|
+
return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
|
480
|
+
}
|
252
481
|
struct BatchControl {
|
253
482
|
FilterStackCall* call_ = nullptr;
|
254
483
|
grpc_transport_stream_op_batch op_;
|
255
|
-
|
256
|
-
|
257
|
-
|
484
|
+
// Share memory for cq_completion and notify_tag as they are never needed
|
485
|
+
// simultaneously. Each byte used in this data structure count as six bytes
|
486
|
+
// per call, so any savings we can make are worthwhile,
|
258
487
|
|
259
|
-
|
260
|
-
|
261
|
-
|
488
|
+
// We use notify_tag to determine whether or not to send notification to the
|
489
|
+
// completion queue. Once we've made that determination, we can reuse the
|
490
|
+
// memory for cq_completion.
|
262
491
|
union {
|
263
492
|
grpc_cq_completion cq_completion;
|
264
493
|
struct {
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
494
|
+
// Any given op indicates completion by either (a) calling a closure or
|
495
|
+
// (b) sending a notification on the call's completion queue. If
|
496
|
+
// \a is_closure is true, \a tag indicates a closure to be invoked;
|
497
|
+
// otherwise, \a tag indicates the tag to be used in the notification to
|
498
|
+
// be sent to the completion queue.
|
270
499
|
void* tag;
|
271
500
|
bool is_closure;
|
272
501
|
} notify_tag;
|
273
502
|
} completion_data_;
|
274
503
|
grpc_closure start_batch_;
|
275
504
|
grpc_closure finish_batch_;
|
276
|
-
std::atomic<intptr_t>
|
505
|
+
std::atomic<intptr_t> ops_pending_{0};
|
277
506
|
AtomicError batch_error_;
|
278
|
-
void
|
279
|
-
|
507
|
+
void set_pending_ops(uintptr_t ops) {
|
508
|
+
ops_pending_.store(ops, std::memory_order_release);
|
280
509
|
}
|
281
|
-
bool completed_batch_step() {
|
282
|
-
|
510
|
+
bool completed_batch_step(PendingOp op) {
|
511
|
+
auto mask = PendingOpMask(op);
|
512
|
+
auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
|
513
|
+
if (grpc_call_trace.enabled()) {
|
514
|
+
gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
|
515
|
+
PendingOpString(mask).c_str(),
|
516
|
+
PendingOpString(r & ~mask).c_str(),
|
517
|
+
completion_data_.notify_tag.tag);
|
518
|
+
}
|
519
|
+
GPR_ASSERT((r & mask) != 0);
|
520
|
+
return r == mask;
|
283
521
|
}
|
284
522
|
|
285
523
|
void PostCompletion();
|
286
|
-
void FinishStep();
|
287
|
-
void ContinueReceivingSlices();
|
288
|
-
void ReceivingSliceReady(grpc_error_handle error);
|
524
|
+
void FinishStep(PendingOp op);
|
289
525
|
void ProcessDataAfterMetadata();
|
290
526
|
void ReceivingStreamReady(grpc_error_handle error);
|
291
527
|
void ValidateFilteredMetadata();
|
@@ -295,9 +531,9 @@ class FilterStackCall final : public Call {
|
|
295
531
|
};
|
296
532
|
|
297
533
|
FilterStackCall(Arena* arena, const grpc_call_create_args& args)
|
298
|
-
: Call(arena, args.server_transport_data == nullptr, args.send_deadline
|
534
|
+
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
535
|
+
args.channel->Ref()),
|
299
536
|
cq_(args.cq),
|
300
|
-
channel_(args.channel),
|
301
537
|
stream_op_payload_(context_) {}
|
302
538
|
|
303
539
|
static void ReleaseCall(void* call, grpc_error_handle);
|
@@ -328,60 +564,55 @@ class FilterStackCall final : public Call {
|
|
328
564
|
CallCombiner call_combiner_;
|
329
565
|
grpc_completion_queue* cq_;
|
330
566
|
grpc_polling_entity pollent_;
|
331
|
-
grpc_channel* channel_;
|
332
567
|
gpr_cycle_counter start_time_ = gpr_get_cycle_counter();
|
333
568
|
|
334
|
-
|
569
|
+
/// has grpc_call_unref been called
|
335
570
|
bool destroy_called_ = false;
|
336
571
|
// Trailers-only response status
|
337
572
|
bool is_trailers_only_ = false;
|
338
|
-
|
573
|
+
/// which ops are in-flight
|
339
574
|
bool sent_initial_metadata_ = false;
|
340
575
|
bool sending_message_ = false;
|
341
576
|
bool sent_final_op_ = false;
|
342
577
|
bool received_initial_metadata_ = false;
|
343
578
|
bool receiving_message_ = false;
|
344
579
|
bool requested_final_op_ = false;
|
345
|
-
gpr_atm any_ops_sent_atm_ = 0;
|
346
580
|
gpr_atm received_final_op_atm_ = 0;
|
347
581
|
|
348
582
|
BatchControl* active_batches_[kMaxConcurrentBatches] = {};
|
349
583
|
grpc_transport_stream_op_batch_payload stream_op_payload_;
|
350
584
|
|
351
|
-
|
585
|
+
// first idx: is_receiving, second idx: is_trailing
|
352
586
|
grpc_metadata_batch send_initial_metadata_{arena()};
|
353
587
|
grpc_metadata_batch send_trailing_metadata_{arena()};
|
354
588
|
grpc_metadata_batch recv_initial_metadata_{arena()};
|
355
589
|
grpc_metadata_batch recv_trailing_metadata_{arena()};
|
356
590
|
|
357
|
-
|
358
|
-
|
591
|
+
// Buffered read metadata waiting to be returned to the application.
|
592
|
+
// Element 0 is initial metadata, element 1 is trailing metadata.
|
359
593
|
grpc_metadata_array* buffered_metadata_[2] = {};
|
360
594
|
|
361
|
-
//
|
362
|
-
|
363
|
-
|
364
|
-
/* Call data useful used for reporting. Only valid after the call has
|
365
|
-
* completed */
|
595
|
+
// Call data useful used for reporting. Only valid after the call has
|
596
|
+
// completed
|
366
597
|
grpc_call_final_info final_info_;
|
367
598
|
|
368
|
-
|
599
|
+
// Compression algorithm for *incoming* data
|
369
600
|
grpc_compression_algorithm incoming_compression_algorithm_ =
|
370
601
|
GRPC_COMPRESS_NONE;
|
371
|
-
|
372
|
-
|
602
|
+
// Supported encodings (compression algorithms), a bitset.
|
603
|
+
// Always support no compression.
|
373
604
|
CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
|
374
605
|
|
375
|
-
|
606
|
+
// Contexts for various subsystems (security, tracing, ...).
|
376
607
|
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
377
608
|
|
378
|
-
|
609
|
+
SliceBuffer send_slice_buffer_;
|
610
|
+
absl::optional<SliceBuffer> receiving_slice_buffer_;
|
611
|
+
uint32_t receiving_stream_flags_;
|
379
612
|
|
380
|
-
OrphanablePtr<ByteStream> receiving_stream_;
|
381
613
|
bool call_failed_before_recv_message_ = false;
|
382
614
|
grpc_byte_buffer** receiving_buffer_ = nullptr;
|
383
615
|
grpc_slice receiving_slice_ = grpc_empty_slice();
|
384
|
-
grpc_closure receiving_slice_ready_;
|
385
616
|
grpc_closure receiving_stream_ready_;
|
386
617
|
grpc_closure receiving_initial_metadata_ready_;
|
387
618
|
grpc_closure receiving_trailing_metadata_ready_;
|
@@ -406,123 +637,50 @@ class FilterStackCall final : public Call {
|
|
406
637
|
} final_op_;
|
407
638
|
AtomicError status_error_;
|
408
639
|
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
640
|
+
// recv_state can contain one of the following values:
|
641
|
+
// RECV_NONE : : no initial metadata and messages received
|
642
|
+
// RECV_INITIAL_METADATA_FIRST : received initial metadata first
|
643
|
+
// a batch_control* : received messages first
|
644
|
+
|
645
|
+
// +------1------RECV_NONE------3-----+
|
646
|
+
// | |
|
647
|
+
// | |
|
648
|
+
// v v
|
649
|
+
// RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
|
650
|
+
// | ^ | ^
|
651
|
+
// | | | |
|
652
|
+
// +-----2-----+ +-----4-----+
|
653
|
+
|
654
|
+
// For 1, 4: See receiving_initial_metadata_ready() function
|
655
|
+
// For 2, 3: See receiving_stream_ready() function
|
425
656
|
gpr_atm recv_state_ = 0;
|
426
657
|
};
|
427
658
|
|
428
|
-
Call::ParentCall* Call::GetOrCreateParentCall() {
|
429
|
-
ParentCall* p = parent_call_.load(std::memory_order_acquire);
|
430
|
-
if (p == nullptr) {
|
431
|
-
p = arena_->New<ParentCall>();
|
432
|
-
ParentCall* expected = nullptr;
|
433
|
-
if (!parent_call_.compare_exchange_strong(expected, p,
|
434
|
-
std::memory_order_release,
|
435
|
-
std::memory_order_relaxed)) {
|
436
|
-
p->~ParentCall();
|
437
|
-
p = expected;
|
438
|
-
}
|
439
|
-
}
|
440
|
-
return p;
|
441
|
-
}
|
442
|
-
|
443
|
-
Call::ParentCall* Call::parent_call() {
|
444
|
-
return parent_call_.load(std::memory_order_acquire);
|
445
|
-
}
|
446
|
-
|
447
|
-
absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
|
448
|
-
child_ = arena()->New<ChildCall>(parent);
|
449
|
-
|
450
|
-
parent->InternalRef("child");
|
451
|
-
GPR_ASSERT(is_client_);
|
452
|
-
GPR_ASSERT(!parent->is_client_);
|
453
|
-
|
454
|
-
if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
|
455
|
-
send_deadline_ = std::min(send_deadline_, parent->send_deadline_);
|
456
|
-
}
|
457
|
-
/* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
|
458
|
-
* GRPC_PROPAGATE_STATS_CONTEXT */
|
459
|
-
/* TODO(ctiller): This should change to use the appropriate census start_op
|
460
|
-
* call. */
|
461
|
-
if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
|
462
|
-
if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
|
463
|
-
return absl::UnknownError(
|
464
|
-
"Census tracing propagation requested without Census context "
|
465
|
-
"propagation");
|
466
|
-
}
|
467
|
-
ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
|
468
|
-
nullptr);
|
469
|
-
} else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
470
|
-
return absl::UnknownError(
|
471
|
-
"Census context propagation requested without Census tracing "
|
472
|
-
"propagation");
|
473
|
-
}
|
474
|
-
if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
|
475
|
-
cancellation_is_inherited_ = true;
|
476
|
-
}
|
477
|
-
return absl::OkStatus();
|
478
|
-
}
|
479
|
-
|
480
|
-
void Call::PublishToParent(Call* parent) {
|
481
|
-
ChildCall* cc = child_;
|
482
|
-
ParentCall* pc = parent->GetOrCreateParentCall();
|
483
|
-
MutexLock lock(&pc->child_list_mu);
|
484
|
-
if (pc->first_child == nullptr) {
|
485
|
-
pc->first_child = this;
|
486
|
-
cc->sibling_next = cc->sibling_prev = this;
|
487
|
-
} else {
|
488
|
-
cc->sibling_next = pc->first_child;
|
489
|
-
cc->sibling_prev = pc->first_child->child_->sibling_prev;
|
490
|
-
cc->sibling_next->child_->sibling_prev =
|
491
|
-
cc->sibling_prev->child_->sibling_next = this;
|
492
|
-
}
|
493
|
-
if (parent->Completed()) {
|
494
|
-
CancelWithError(GRPC_ERROR_CANCELLED);
|
495
|
-
}
|
496
|
-
}
|
497
|
-
|
498
659
|
grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
499
660
|
grpc_call** out_call) {
|
500
|
-
|
501
|
-
|
502
|
-
GRPC_CHANNEL_INTERNAL_REF(args->channel, "call");
|
661
|
+
Channel* channel = args->channel.get();
|
503
662
|
|
504
663
|
auto add_init_error = [](grpc_error_handle* composite,
|
505
664
|
grpc_error_handle new_err) {
|
506
|
-
if (new_err
|
507
|
-
if (
|
508
|
-
*composite =
|
665
|
+
if (new_err.ok()) return;
|
666
|
+
if (composite->ok()) {
|
667
|
+
*composite = GRPC_ERROR_CREATE("Call creation failed");
|
509
668
|
}
|
510
669
|
*composite = grpc_error_add_child(*composite, new_err);
|
511
670
|
};
|
512
671
|
|
513
672
|
Arena* arena;
|
514
673
|
FilterStackCall* call;
|
515
|
-
grpc_error_handle error
|
516
|
-
grpc_channel_stack* channel_stack =
|
517
|
-
|
518
|
-
|
519
|
-
GRPC_STATS_INC_CALL_INITIAL_SIZE(initial_size);
|
674
|
+
grpc_error_handle error;
|
675
|
+
grpc_channel_stack* channel_stack = channel->channel_stack();
|
676
|
+
size_t initial_size = channel->CallSizeEstimate();
|
677
|
+
global_stats().IncrementCallInitialSize(initial_size);
|
520
678
|
size_t call_alloc_size =
|
521
679
|
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
|
522
680
|
channel_stack->call_stack_size;
|
523
681
|
|
524
682
|
std::pair<Arena*, void*> arena_with_call = Arena::CreateWithAlloc(
|
525
|
-
initial_size, call_alloc_size,
|
683
|
+
initial_size, call_alloc_size, channel->allocator());
|
526
684
|
arena = arena_with_call.first;
|
527
685
|
call = new (arena_with_call.second) FilterStackCall(arena, *args);
|
528
686
|
GPR_DEBUG_ASSERT(FromC(call->c_ptr()) == call);
|
@@ -533,8 +691,8 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
533
691
|
call->final_op_.client.status_details = nullptr;
|
534
692
|
call->final_op_.client.status = nullptr;
|
535
693
|
call->final_op_.client.error_string = nullptr;
|
536
|
-
|
537
|
-
path =
|
694
|
+
global_stats().IncrementClientCallsCreated();
|
695
|
+
path = CSliceRef(args->path->c_slice());
|
538
696
|
call->send_initial_metadata_.Set(HttpPathMetadata(),
|
539
697
|
std::move(*args->path));
|
540
698
|
if (args->authority.has_value()) {
|
@@ -542,7 +700,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
542
700
|
std::move(*args->authority));
|
543
701
|
}
|
544
702
|
} else {
|
545
|
-
|
703
|
+
global_stats().IncrementServerCallsCreated();
|
546
704
|
call->final_op_.server.cancelled = nullptr;
|
547
705
|
call->final_op_.server.core_server = args->server;
|
548
706
|
}
|
@@ -552,7 +710,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
552
710
|
add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
|
553
711
|
parent, args->propagation_mask)));
|
554
712
|
}
|
555
|
-
|
713
|
+
// initial refcount dropped by grpc_call_unref
|
556
714
|
grpc_call_element_args call_args = {
|
557
715
|
call->call_stack(), args->server_transport_data,
|
558
716
|
call->context_, path,
|
@@ -565,8 +723,8 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
565
723
|
call->PublishToParent(parent);
|
566
724
|
}
|
567
725
|
|
568
|
-
if (error
|
569
|
-
call->CancelWithError(
|
726
|
+
if (!error.ok()) {
|
727
|
+
call->CancelWithError(error);
|
570
728
|
}
|
571
729
|
if (args->cq != nullptr) {
|
572
730
|
GPR_ASSERT(args->pollset_set_alternative == nullptr &&
|
@@ -586,8 +744,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
586
744
|
}
|
587
745
|
|
588
746
|
if (call->is_client()) {
|
589
|
-
channelz::ChannelNode* channelz_channel =
|
590
|
-
grpc_channel_get_channelz_node(call->channel_);
|
747
|
+
channelz::ChannelNode* channelz_channel = channel->channelz_node();
|
591
748
|
if (channelz_channel != nullptr) {
|
592
749
|
channelz_channel->RecordCallStarted();
|
593
750
|
}
|
@@ -599,7 +756,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
599
756
|
}
|
600
757
|
}
|
601
758
|
|
602
|
-
|
759
|
+
CSliceUnref(path);
|
603
760
|
|
604
761
|
return error;
|
605
762
|
}
|
@@ -608,8 +765,7 @@ void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
|
608
765
|
GPR_ASSERT(cq);
|
609
766
|
|
610
767
|
if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
|
611
|
-
|
612
|
-
abort();
|
768
|
+
Crash("A pollset_set is already registered for this call.");
|
613
769
|
}
|
614
770
|
cq_ = cq;
|
615
771
|
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
@@ -618,20 +774,14 @@ void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
|
618
774
|
}
|
619
775
|
|
620
776
|
void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
|
621
|
-
|
622
|
-
grpc_channel* channel = c->channel_;
|
623
|
-
Arena* arena = c->arena();
|
624
|
-
c->~FilterStackCall();
|
625
|
-
grpc_channel_update_call_size_estimate(channel, arena->Destroy());
|
626
|
-
GRPC_CHANNEL_INTERNAL_UNREF(channel, "call");
|
777
|
+
static_cast<FilterStackCall*>(call)->DeleteThis();
|
627
778
|
}
|
628
779
|
|
629
780
|
void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
630
|
-
GPR_TIMER_SCOPE("destroy_call", 0);
|
631
781
|
auto* c = static_cast<FilterStackCall*>(call);
|
632
782
|
c->recv_initial_metadata_.Clear();
|
633
783
|
c->recv_trailing_metadata_.Clear();
|
634
|
-
c->
|
784
|
+
c->receiving_slice_buffer_.reset();
|
635
785
|
ParentCall* pc = c->parent_call();
|
636
786
|
if (pc != nullptr) {
|
637
787
|
pc->~ParentCall();
|
@@ -644,7 +794,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
|
644
794
|
grpc_error_get_status(status_error, c->send_deadline(),
|
645
795
|
&c->final_info_.final_status, nullptr, nullptr,
|
646
796
|
&(c->final_info_.error_string));
|
647
|
-
c->status_error_.set(
|
797
|
+
c->status_error_.set(absl::OkStatus());
|
648
798
|
c->final_info_.stats.latency =
|
649
799
|
gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time_);
|
650
800
|
grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
|
@@ -652,30 +802,9 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
|
652
802
|
grpc_schedule_on_exec_ctx));
|
653
803
|
}
|
654
804
|
|
655
|
-
void Call::MaybeUnpublishFromParent() {
|
656
|
-
ChildCall* cc = child_;
|
657
|
-
if (cc == nullptr) return;
|
658
|
-
|
659
|
-
ParentCall* pc = cc->parent->parent_call();
|
660
|
-
{
|
661
|
-
MutexLock lock(&pc->child_list_mu);
|
662
|
-
if (this == pc->first_child) {
|
663
|
-
pc->first_child = cc->sibling_next;
|
664
|
-
if (this == pc->first_child) {
|
665
|
-
pc->first_child = nullptr;
|
666
|
-
}
|
667
|
-
}
|
668
|
-
cc->sibling_prev->child_->sibling_next = cc->sibling_next;
|
669
|
-
cc->sibling_next->child_->sibling_prev = cc->sibling_prev;
|
670
|
-
}
|
671
|
-
cc->parent->InternalUnref("child");
|
672
|
-
}
|
673
|
-
|
674
805
|
void FilterStackCall::ExternalUnref() {
|
675
806
|
if (GPR_LIKELY(!ext_ref_.Unref())) return;
|
676
807
|
|
677
|
-
GPR_TIMER_SCOPE("grpc_call_unref", 0);
|
678
|
-
|
679
808
|
ApplicationCallbackExecCtx callback_exec_ctx;
|
680
809
|
ExecCtx exec_ctx;
|
681
810
|
|
@@ -685,10 +814,9 @@ void FilterStackCall::ExternalUnref() {
|
|
685
814
|
|
686
815
|
GPR_ASSERT(!destroy_called_);
|
687
816
|
destroy_called_ = true;
|
688
|
-
bool cancel = gpr_atm_acq_load(&
|
689
|
-
gpr_atm_acq_load(&received_final_op_atm_) == 0;
|
817
|
+
bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
|
690
818
|
if (cancel) {
|
691
|
-
CancelWithError(
|
819
|
+
CancelWithError(absl::CancelledError());
|
692
820
|
} else {
|
693
821
|
// Unset the call combiner cancellation closure. This has the
|
694
822
|
// effect of scheduling the previously set cancellation closure, if
|
@@ -699,14 +827,6 @@ void FilterStackCall::ExternalUnref() {
|
|
699
827
|
InternalUnref("destroy");
|
700
828
|
}
|
701
829
|
|
702
|
-
char* FilterStackCall::GetPeer() {
|
703
|
-
char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
|
704
|
-
if (peer_string != nullptr) return gpr_strdup(peer_string);
|
705
|
-
peer_string = grpc_channel_get_target(channel_);
|
706
|
-
if (peer_string != nullptr) return peer_string;
|
707
|
-
return gpr_strdup("unknown");
|
708
|
-
}
|
709
|
-
|
710
830
|
// start_batch_closure points to a caller-allocated closure to be used
|
711
831
|
// for entering the call combiner.
|
712
832
|
void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
@@ -714,7 +834,6 @@ void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
|
714
834
|
// This is called via the call combiner to start sending a batch down
|
715
835
|
// the filter stack.
|
716
836
|
auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
|
717
|
-
GPR_TIMER_SCOPE("execute_batch_in_call_combiner", 0);
|
718
837
|
grpc_transport_stream_op_batch* batch =
|
719
838
|
static_cast<grpc_transport_stream_op_batch*>(arg);
|
720
839
|
auto* call =
|
@@ -727,7 +846,7 @@ void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
|
727
846
|
GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
|
728
847
|
grpc_schedule_on_exec_ctx);
|
729
848
|
GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
|
730
|
-
|
849
|
+
absl::OkStatus(), "executing batch");
|
731
850
|
}
|
732
851
|
|
733
852
|
namespace {
|
@@ -750,15 +869,15 @@ static void done_termination(void* arg, grpc_error_handle /*error*/) {
|
|
750
869
|
|
751
870
|
void FilterStackCall::CancelWithError(grpc_error_handle error) {
|
752
871
|
if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
|
753
|
-
GRPC_ERROR_UNREF(error);
|
754
872
|
return;
|
755
873
|
}
|
874
|
+
ClearPeerString();
|
756
875
|
InternalRef("termination");
|
757
876
|
// Inform the call combiner of the cancellation, so that it can cancel
|
758
877
|
// any in-flight asynchronous actions that may be holding the call
|
759
878
|
// combiner. This ensures that the cancel_stream batch can be sent
|
760
879
|
// down the filter stack in a timely manner.
|
761
|
-
call_combiner_.Cancel(
|
880
|
+
call_combiner_.Cancel(error);
|
762
881
|
CancelState* state = new CancelState;
|
763
882
|
state->call = this;
|
764
883
|
GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
|
@@ -770,19 +889,10 @@ void FilterStackCall::CancelWithError(grpc_error_handle error) {
|
|
770
889
|
ExecuteBatch(op, &state->start_batch);
|
771
890
|
}
|
772
891
|
|
773
|
-
void Call::CancelWithStatus(grpc_status_code status, const char* description) {
|
774
|
-
// copying 'description' is needed to ensure the grpc_call_cancel_with_status
|
775
|
-
// guarantee that can be short-lived.
|
776
|
-
CancelWithError(grpc_error_set_int(
|
777
|
-
grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description),
|
778
|
-
GRPC_ERROR_STR_GRPC_MESSAGE, description),
|
779
|
-
GRPC_ERROR_INT_GRPC_STATUS, status));
|
780
|
-
}
|
781
|
-
|
782
892
|
void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
783
893
|
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
784
894
|
gpr_log(GPR_DEBUG, "set_final_status %s", is_client() ? "CLI" : "SVR");
|
785
|
-
gpr_log(GPR_DEBUG, "%s",
|
895
|
+
gpr_log(GPR_DEBUG, "%s", StatusToString(error).c_str());
|
786
896
|
}
|
787
897
|
if (is_client()) {
|
788
898
|
std::string status_details;
|
@@ -792,9 +902,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
|
792
902
|
*final_op_.client.status_details =
|
793
903
|
grpc_slice_from_cpp_string(std::move(status_details));
|
794
904
|
status_error_.set(error);
|
795
|
-
|
796
|
-
channelz::ChannelNode* channelz_channel =
|
797
|
-
grpc_channel_get_channelz_node(channel_);
|
905
|
+
channelz::ChannelNode* channelz_channel = channel()->channelz_node();
|
798
906
|
if (channelz_channel != nullptr) {
|
799
907
|
if (*final_op_.client.status != GRPC_STATUS_OK) {
|
800
908
|
channelz_channel->RecordCallFailed();
|
@@ -804,7 +912,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
|
804
912
|
}
|
805
913
|
} else {
|
806
914
|
*final_op_.server.cancelled =
|
807
|
-
error
|
915
|
+
!error.ok() || !sent_server_trailing_metadata_;
|
808
916
|
channelz::ServerNode* channelz_node =
|
809
917
|
final_op_.server.core_server->channelz_node();
|
810
918
|
if (channelz_node != nullptr) {
|
@@ -814,7 +922,6 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
|
814
922
|
channelz_node->RecordCallSucceeded();
|
815
923
|
}
|
816
924
|
}
|
817
|
-
GRPC_ERROR_UNREF(error);
|
818
925
|
}
|
819
926
|
}
|
820
927
|
|
@@ -840,8 +947,7 @@ bool FilterStackCall::PrepareApplicationMetadata(size_t count,
|
|
840
947
|
// Filter "content-length metadata"
|
841
948
|
continue;
|
842
949
|
}
|
843
|
-
batch->Append(StringViewFromSlice(md->key),
|
844
|
-
Slice(grpc_slice_ref_internal(md->value)),
|
950
|
+
batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
|
845
951
|
[md](absl::string_view error, const Slice& value) {
|
846
952
|
gpr_log(GPR_DEBUG, "Append error: %s",
|
847
953
|
absl::StrCat("key=", StringViewFromSlice(md->key),
|
@@ -914,7 +1020,6 @@ void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
|
|
914
1020
|
if (b->count() == 0) return;
|
915
1021
|
if (!is_client() && is_trailing) return;
|
916
1022
|
if (is_trailing && buffered_metadata_[1] == nullptr) return;
|
917
|
-
GPR_TIMER_SCOPE("publish_app_metadata", 0);
|
918
1023
|
grpc_metadata_array* dest;
|
919
1024
|
dest = buffered_metadata_[is_trailing];
|
920
1025
|
if (dest->count + b->count() > dest->capacity) {
|
@@ -938,39 +1043,37 @@ void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
|
|
938
1043
|
|
939
1044
|
void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
940
1045
|
grpc_error_handle batch_error) {
|
941
|
-
if (batch_error
|
1046
|
+
if (!batch_error.ok()) {
|
942
1047
|
SetFinalStatus(batch_error);
|
943
1048
|
} else {
|
944
1049
|
absl::optional<grpc_status_code> grpc_status =
|
945
1050
|
b->Take(GrpcStatusMetadata());
|
946
1051
|
if (grpc_status.has_value()) {
|
947
1052
|
grpc_status_code status_code = *grpc_status;
|
948
|
-
grpc_error_handle error
|
1053
|
+
grpc_error_handle error;
|
949
1054
|
if (status_code != GRPC_STATUS_OK) {
|
950
1055
|
char* peer = GetPeer();
|
951
1056
|
error = grpc_error_set_int(
|
952
|
-
|
953
|
-
|
954
|
-
GRPC_ERROR_INT_GRPC_STATUS, static_cast<intptr_t>(status_code));
|
1057
|
+
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ", peer)),
|
1058
|
+
StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
|
955
1059
|
gpr_free(peer);
|
956
1060
|
}
|
957
1061
|
auto grpc_message = b->Take(GrpcMessageMetadata());
|
958
1062
|
if (grpc_message.has_value()) {
|
959
|
-
error = grpc_error_set_str(error,
|
1063
|
+
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
|
960
1064
|
grpc_message->as_string_view());
|
961
|
-
} else if (error
|
962
|
-
error = grpc_error_set_str(error,
|
1065
|
+
} else if (!error.ok()) {
|
1066
|
+
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
|
963
1067
|
}
|
964
|
-
SetFinalStatus(
|
965
|
-
GRPC_ERROR_UNREF(error);
|
1068
|
+
SetFinalStatus(error);
|
966
1069
|
} else if (!is_client()) {
|
967
|
-
SetFinalStatus(
|
1070
|
+
SetFinalStatus(absl::OkStatus());
|
968
1071
|
} else {
|
969
1072
|
gpr_log(GPR_DEBUG,
|
970
1073
|
"Received trailing metadata with no error and no status");
|
971
|
-
SetFinalStatus(grpc_error_set_int(
|
972
|
-
|
973
|
-
|
1074
|
+
SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
|
1075
|
+
StatusIntProperty::kRpcStatus,
|
1076
|
+
GRPC_STATUS_UNKNOWN));
|
974
1077
|
}
|
975
1078
|
}
|
976
1079
|
PublishAppMetadata(b, true);
|
@@ -978,7 +1081,7 @@ void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
|
978
1081
|
|
979
1082
|
namespace {
|
980
1083
|
bool AreWriteFlagsValid(uint32_t flags) {
|
981
|
-
|
1084
|
+
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
982
1085
|
const uint32_t allowed_write_positions =
|
983
1086
|
(GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
|
984
1087
|
const uint32_t invalid_positions = ~allowed_write_positions;
|
@@ -986,7 +1089,7 @@ bool AreWriteFlagsValid(uint32_t flags) {
|
|
986
1089
|
}
|
987
1090
|
|
988
1091
|
bool AreInitialMetadataFlagsValid(uint32_t flags) {
|
989
|
-
|
1092
|
+
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
990
1093
|
uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
|
991
1094
|
return !(flags & invalid_positions);
|
992
1095
|
}
|
@@ -1034,68 +1137,48 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
|
|
1034
1137
|
return bctl;
|
1035
1138
|
}
|
1036
1139
|
|
1037
|
-
void Call::PropagateCancellationToChildren() {
|
1038
|
-
ParentCall* pc = parent_call();
|
1039
|
-
if (pc != nullptr) {
|
1040
|
-
Call* child;
|
1041
|
-
MutexLock lock(&pc->child_list_mu);
|
1042
|
-
child = pc->first_child;
|
1043
|
-
if (child != nullptr) {
|
1044
|
-
do {
|
1045
|
-
Call* next_child_call = child->child_->sibling_next;
|
1046
|
-
if (child->cancellation_is_inherited_) {
|
1047
|
-
child->InternalRef("propagate_cancel");
|
1048
|
-
child->CancelWithError(GRPC_ERROR_CANCELLED);
|
1049
|
-
child->InternalUnref("propagate_cancel");
|
1050
|
-
}
|
1051
|
-
child = next_child_call;
|
1052
|
-
} while (child != pc->first_child);
|
1053
|
-
}
|
1054
|
-
}
|
1055
|
-
}
|
1056
|
-
|
1057
1140
|
void FilterStackCall::BatchControl::PostCompletion() {
|
1058
1141
|
FilterStackCall* call = call_;
|
1059
|
-
grpc_error_handle error =
|
1142
|
+
grpc_error_handle error = batch_error_.get();
|
1143
|
+
if (grpc_call_trace.enabled()) {
|
1144
|
+
gpr_log(GPR_DEBUG, "tag:%p batch_error=%s", completion_data_.notify_tag.tag,
|
1145
|
+
error.ToString().c_str());
|
1146
|
+
}
|
1060
1147
|
|
1061
1148
|
if (op_.send_initial_metadata) {
|
1062
1149
|
call->send_initial_metadata_.Clear();
|
1063
1150
|
}
|
1064
1151
|
if (op_.send_message) {
|
1065
|
-
if (op_.payload->send_message.stream_write_closed &&
|
1066
|
-
error == GRPC_ERROR_NONE) {
|
1152
|
+
if (op_.payload->send_message.stream_write_closed && error.ok()) {
|
1067
1153
|
error = grpc_error_add_child(
|
1068
|
-
error,
|
1154
|
+
error, GRPC_ERROR_CREATE(
|
1069
1155
|
"Attempt to send message after stream was closed."));
|
1070
1156
|
}
|
1071
1157
|
call->sending_message_ = false;
|
1158
|
+
call->send_slice_buffer_.Clear();
|
1072
1159
|
}
|
1073
1160
|
if (op_.send_trailing_metadata) {
|
1074
1161
|
call->send_trailing_metadata_.Clear();
|
1075
1162
|
}
|
1076
1163
|
if (op_.recv_trailing_metadata) {
|
1077
|
-
|
1164
|
+
// propagate cancellation to any interested children
|
1078
1165
|
gpr_atm_rel_store(&call->received_final_op_atm_, 1);
|
1079
1166
|
call->PropagateCancellationToChildren();
|
1080
|
-
|
1081
|
-
error = GRPC_ERROR_NONE;
|
1167
|
+
error = absl::OkStatus();
|
1082
1168
|
}
|
1083
|
-
if (error
|
1084
|
-
*call->receiving_buffer_ != nullptr) {
|
1169
|
+
if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
|
1085
1170
|
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1086
1171
|
*call->receiving_buffer_ = nullptr;
|
1087
1172
|
}
|
1088
|
-
batch_error_.set(
|
1173
|
+
batch_error_.set(absl::OkStatus());
|
1089
1174
|
|
1090
1175
|
if (completion_data_.notify_tag.is_closure) {
|
1091
|
-
/* unrefs error */
|
1092
1176
|
call_ = nullptr;
|
1093
1177
|
Closure::Run(DEBUG_LOCATION,
|
1094
1178
|
static_cast<grpc_closure*>(completion_data_.notify_tag.tag),
|
1095
1179
|
error);
|
1096
1180
|
call->InternalUnref("completion");
|
1097
1181
|
} else {
|
1098
|
-
/* unrefs error */
|
1099
1182
|
grpc_cq_end_op(
|
1100
1183
|
call->cq_, completion_data_.notify_tag.tag, error,
|
1101
1184
|
[](void* user_data, grpc_cq_completion* /*storage*/) {
|
@@ -1108,118 +1191,58 @@ void FilterStackCall::BatchControl::PostCompletion() {
|
|
1108
1191
|
}
|
1109
1192
|
}
|
1110
1193
|
|
1111
|
-
void FilterStackCall::BatchControl::FinishStep() {
|
1112
|
-
if (GPR_UNLIKELY(completed_batch_step())) {
|
1194
|
+
void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
|
1195
|
+
if (GPR_UNLIKELY(completed_batch_step(op))) {
|
1113
1196
|
PostCompletion();
|
1114
1197
|
}
|
1115
1198
|
}
|
1116
1199
|
|
1117
|
-
void FilterStackCall::BatchControl::ContinueReceivingSlices() {
|
1118
|
-
grpc_error_handle error;
|
1119
|
-
FilterStackCall* call = call_;
|
1120
|
-
for (;;) {
|
1121
|
-
size_t remaining = call->receiving_stream_->length() -
|
1122
|
-
(*call->receiving_buffer_)->data.raw.slice_buffer.length;
|
1123
|
-
if (remaining == 0) {
|
1124
|
-
call->receiving_message_ = false;
|
1125
|
-
call->receiving_stream_.reset();
|
1126
|
-
FinishStep();
|
1127
|
-
return;
|
1128
|
-
}
|
1129
|
-
if (call->receiving_stream_->Next(remaining,
|
1130
|
-
&call->receiving_slice_ready_)) {
|
1131
|
-
error = call->receiving_stream_->Pull(&call->receiving_slice_);
|
1132
|
-
if (error == GRPC_ERROR_NONE) {
|
1133
|
-
grpc_slice_buffer_add(
|
1134
|
-
&(*call->receiving_buffer_)->data.raw.slice_buffer,
|
1135
|
-
call->receiving_slice_);
|
1136
|
-
} else {
|
1137
|
-
call->receiving_stream_.reset();
|
1138
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1139
|
-
*call->receiving_buffer_ = nullptr;
|
1140
|
-
call->receiving_message_ = false;
|
1141
|
-
FinishStep();
|
1142
|
-
GRPC_ERROR_UNREF(error);
|
1143
|
-
return;
|
1144
|
-
}
|
1145
|
-
} else {
|
1146
|
-
return;
|
1147
|
-
}
|
1148
|
-
}
|
1149
|
-
}
|
1150
|
-
|
1151
|
-
void FilterStackCall::BatchControl::ReceivingSliceReady(
|
1152
|
-
grpc_error_handle error) {
|
1153
|
-
FilterStackCall* call = call_;
|
1154
|
-
bool release_error = false;
|
1155
|
-
|
1156
|
-
if (error == GRPC_ERROR_NONE) {
|
1157
|
-
grpc_slice slice;
|
1158
|
-
error = call->receiving_stream_->Pull(&slice);
|
1159
|
-
if (error == GRPC_ERROR_NONE) {
|
1160
|
-
grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer,
|
1161
|
-
slice);
|
1162
|
-
ContinueReceivingSlices();
|
1163
|
-
} else {
|
1164
|
-
/* Error returned by ByteStream::Pull() needs to be released manually */
|
1165
|
-
release_error = true;
|
1166
|
-
}
|
1167
|
-
}
|
1168
|
-
|
1169
|
-
if (error != GRPC_ERROR_NONE) {
|
1170
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
|
1171
|
-
GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
|
1172
|
-
}
|
1173
|
-
call->receiving_stream_.reset();
|
1174
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1175
|
-
*call->receiving_buffer_ = nullptr;
|
1176
|
-
call->receiving_message_ = false;
|
1177
|
-
FinishStep();
|
1178
|
-
if (release_error) {
|
1179
|
-
GRPC_ERROR_UNREF(error);
|
1180
|
-
}
|
1181
|
-
}
|
1182
|
-
}
|
1183
|
-
|
1184
1200
|
void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
|
1185
1201
|
FilterStackCall* call = call_;
|
1186
|
-
if (call->
|
1202
|
+
if (!call->receiving_slice_buffer_.has_value()) {
|
1187
1203
|
*call->receiving_buffer_ = nullptr;
|
1188
1204
|
call->receiving_message_ = false;
|
1189
|
-
FinishStep();
|
1205
|
+
FinishStep(PendingOp::kRecvMessage);
|
1190
1206
|
} else {
|
1191
|
-
call->test_only_last_message_flags_ = call->
|
1192
|
-
if ((call->
|
1207
|
+
call->test_only_last_message_flags_ = call->receiving_stream_flags_;
|
1208
|
+
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
1193
1209
|
(call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
|
1194
1210
|
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
|
1195
1211
|
nullptr, 0, call->incoming_compression_algorithm_);
|
1196
1212
|
} else {
|
1197
1213
|
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
1198
1214
|
}
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
ContinueReceivingSlices();
|
1215
|
+
grpc_slice_buffer_move_into(
|
1216
|
+
call->receiving_slice_buffer_->c_slice_buffer(),
|
1217
|
+
&(*call->receiving_buffer_)->data.raw.slice_buffer);
|
1218
|
+
call->receiving_message_ = false;
|
1219
|
+
call->receiving_slice_buffer_.reset();
|
1220
|
+
FinishStep(PendingOp::kRecvMessage);
|
1206
1221
|
}
|
1207
1222
|
}
|
1208
1223
|
|
1209
1224
|
void FilterStackCall::BatchControl::ReceivingStreamReady(
|
1210
1225
|
grpc_error_handle error) {
|
1226
|
+
if (grpc_call_trace.enabled()) {
|
1227
|
+
gpr_log(GPR_DEBUG,
|
1228
|
+
"tag:%p ReceivingStreamReady error=%s "
|
1229
|
+
"receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
|
1230
|
+
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1231
|
+
call_->receiving_slice_buffer_.has_value(),
|
1232
|
+
gpr_atm_no_barrier_load(&call_->recv_state_));
|
1233
|
+
}
|
1211
1234
|
FilterStackCall* call = call_;
|
1212
|
-
if (error
|
1213
|
-
call->
|
1235
|
+
if (!error.ok()) {
|
1236
|
+
call->receiving_slice_buffer_.reset();
|
1214
1237
|
if (batch_error_.ok()) {
|
1215
1238
|
batch_error_.set(error);
|
1216
1239
|
}
|
1217
|
-
call->CancelWithError(
|
1240
|
+
call->CancelWithError(error);
|
1218
1241
|
}
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
if (error
|
1242
|
+
// If recv_state is kRecvNone, we will save the batch_control
|
1243
|
+
// object with rel_cas, and will not use it after the cas. Its corresponding
|
1244
|
+
// acq_load is in receiving_initial_metadata_ready()
|
1245
|
+
if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
|
1223
1246
|
!gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
|
1224
1247
|
reinterpret_cast<gpr_atm>(this))) {
|
1225
1248
|
ProcessDataAfterMetadata();
|
@@ -1243,23 +1266,24 @@ void FilterStackCall::HandleCompressionAlgorithmNotAccepted(
|
|
1243
1266
|
gpr_log(GPR_ERROR,
|
1244
1267
|
"Compression algorithm ('%s') not present in the "
|
1245
1268
|
"accepted encodings (%s)",
|
1246
|
-
algo_name,
|
1269
|
+
algo_name,
|
1270
|
+
std::string(encodings_accepted_by_peer_.ToString()).c_str());
|
1247
1271
|
}
|
1248
1272
|
|
1249
1273
|
void FilterStackCall::BatchControl::ValidateFilteredMetadata() {
|
1250
1274
|
FilterStackCall* call = call_;
|
1251
1275
|
|
1252
1276
|
const grpc_compression_options compression_options =
|
1253
|
-
|
1277
|
+
call->channel()->compression_options();
|
1254
1278
|
const grpc_compression_algorithm compression_algorithm =
|
1255
1279
|
call->incoming_compression_algorithm_;
|
1256
1280
|
if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
|
1257
1281
|
compression_options.enabled_algorithms_bitset)
|
1258
1282
|
.IsSet(compression_algorithm))) {
|
1259
|
-
|
1283
|
+
// check if algorithm is supported by current channel config
|
1260
1284
|
call->HandleCompressionAlgorithmDisabled(compression_algorithm);
|
1261
1285
|
}
|
1262
|
-
|
1286
|
+
// GRPC_COMPRESS_NONE is always set.
|
1263
1287
|
GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
1264
1288
|
if (GPR_UNLIKELY(
|
1265
1289
|
!call->encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
@@ -1275,12 +1299,11 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
|
1275
1299
|
|
1276
1300
|
GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
|
1277
1301
|
|
1278
|
-
if (error
|
1302
|
+
if (error.ok()) {
|
1279
1303
|
grpc_metadata_batch* md = &call->recv_initial_metadata_;
|
1280
1304
|
call->RecvInitialFilter(md);
|
1281
1305
|
|
1282
|
-
|
1283
|
-
GPR_TIMER_SCOPE("validate_filtered_metadata", 0);
|
1306
|
+
// TODO(ctiller): this could be moved into recv_initial_filter now
|
1284
1307
|
ValidateFilteredMetadata();
|
1285
1308
|
|
1286
1309
|
absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
|
@@ -1291,41 +1314,41 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
|
1291
1314
|
if (batch_error_.ok()) {
|
1292
1315
|
batch_error_.set(error);
|
1293
1316
|
}
|
1294
|
-
call->CancelWithError(
|
1317
|
+
call->CancelWithError(error);
|
1295
1318
|
}
|
1296
1319
|
|
1297
1320
|
grpc_closure* saved_rsr_closure = nullptr;
|
1298
1321
|
while (true) {
|
1299
1322
|
gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
|
1300
|
-
|
1323
|
+
// Should only receive initial metadata once
|
1301
1324
|
GPR_ASSERT(rsr_bctlp != 1);
|
1302
1325
|
if (rsr_bctlp == 0) {
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1326
|
+
// We haven't seen initial metadata and messages before, thus initial
|
1327
|
+
// metadata is received first.
|
1328
|
+
// no_barrier_cas is used, as this function won't access the batch_control
|
1329
|
+
// object saved by receiving_stream_ready() if the initial metadata is
|
1330
|
+
// received first.
|
1308
1331
|
if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
|
1309
1332
|
kRecvInitialMetadataFirst)) {
|
1310
1333
|
break;
|
1311
1334
|
}
|
1312
1335
|
} else {
|
1313
|
-
|
1336
|
+
// Already received messages
|
1314
1337
|
saved_rsr_closure = GRPC_CLOSURE_CREATE(
|
1315
1338
|
[](void* bctl, grpc_error_handle error) {
|
1316
1339
|
static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
|
1317
1340
|
},
|
1318
1341
|
reinterpret_cast<BatchControl*>(rsr_bctlp),
|
1319
1342
|
grpc_schedule_on_exec_ctx);
|
1320
|
-
|
1343
|
+
// No need to modify recv_state
|
1321
1344
|
break;
|
1322
1345
|
}
|
1323
1346
|
}
|
1324
1347
|
if (saved_rsr_closure != nullptr) {
|
1325
|
-
Closure::Run(DEBUG_LOCATION, saved_rsr_closure,
|
1348
|
+
Closure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
|
1326
1349
|
}
|
1327
1350
|
|
1328
|
-
FinishStep();
|
1351
|
+
FinishStep(PendingOp::kRecvInitialMetadata);
|
1329
1352
|
}
|
1330
1353
|
|
1331
1354
|
void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
|
@@ -1333,8 +1356,8 @@ void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
|
|
1333
1356
|
GRPC_CALL_COMBINER_STOP(call_->call_combiner(),
|
1334
1357
|
"recv_trailing_metadata_ready");
|
1335
1358
|
grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
|
1336
|
-
call_->RecvTrailingFilter(md,
|
1337
|
-
FinishStep();
|
1359
|
+
call_->RecvTrailingFilter(md, error);
|
1360
|
+
FinishStep(PendingOp::kRecvTrailingMetadata);
|
1338
1361
|
}
|
1339
1362
|
|
1340
1363
|
void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
|
@@ -1342,26 +1365,41 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
|
|
1342
1365
|
if (batch_error_.ok()) {
|
1343
1366
|
batch_error_.set(error);
|
1344
1367
|
}
|
1345
|
-
if (error
|
1346
|
-
call_->CancelWithError(
|
1368
|
+
if (!error.ok()) {
|
1369
|
+
call_->CancelWithError(error);
|
1370
|
+
}
|
1371
|
+
FinishStep(PendingOp::kSends);
|
1372
|
+
}
|
1373
|
+
|
1374
|
+
namespace {
|
1375
|
+
void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag,
|
1376
|
+
bool is_notify_tag_closure) {
|
1377
|
+
if (!is_notify_tag_closure) {
|
1378
|
+
GPR_ASSERT(grpc_cq_begin_op(cq, notify_tag));
|
1379
|
+
grpc_cq_end_op(
|
1380
|
+
cq, notify_tag, absl::OkStatus(),
|
1381
|
+
[](void*, grpc_cq_completion* completion) { gpr_free(completion); },
|
1382
|
+
nullptr,
|
1383
|
+
static_cast<grpc_cq_completion*>(
|
1384
|
+
gpr_malloc(sizeof(grpc_cq_completion))));
|
1385
|
+
} else {
|
1386
|
+
Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
|
1387
|
+
absl::OkStatus());
|
1347
1388
|
}
|
1348
|
-
FinishStep();
|
1349
1389
|
}
|
1390
|
+
} // namespace
|
1350
1391
|
|
1351
1392
|
grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
1352
1393
|
void* notify_tag,
|
1353
1394
|
bool is_notify_tag_closure) {
|
1354
|
-
GPR_TIMER_SCOPE("call_start_batch", 0);
|
1355
|
-
|
1356
1395
|
size_t i;
|
1357
1396
|
const grpc_op* op;
|
1358
1397
|
BatchControl* bctl;
|
1359
|
-
bool has_send_ops = false;
|
1360
|
-
int num_recv_ops = 0;
|
1361
1398
|
grpc_call_error error = GRPC_CALL_OK;
|
1362
1399
|
grpc_transport_stream_op_batch* stream_op;
|
1363
1400
|
grpc_transport_stream_op_batch_payload* stream_op_payload;
|
1364
1401
|
uint32_t seen_ops = 0;
|
1402
|
+
intptr_t pending_ops = 0;
|
1365
1403
|
|
1366
1404
|
for (i = 0; i < nops; i++) {
|
1367
1405
|
if (seen_ops & (1u << ops[i].op)) {
|
@@ -1370,21 +1408,19 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1370
1408
|
seen_ops |= (1u << ops[i].op);
|
1371
1409
|
}
|
1372
1410
|
|
1411
|
+
if (!is_client() &&
|
1412
|
+
(seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
|
1413
|
+
(seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
|
1414
|
+
gpr_log(GPR_ERROR,
|
1415
|
+
"******************* SEND_STATUS WITH RECV_MESSAGE "
|
1416
|
+
"*******************");
|
1417
|
+
return GRPC_CALL_ERROR;
|
1418
|
+
}
|
1419
|
+
|
1373
1420
|
GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
|
1374
1421
|
|
1375
1422
|
if (nops == 0) {
|
1376
|
-
|
1377
|
-
GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag));
|
1378
|
-
grpc_cq_end_op(
|
1379
|
-
cq_, notify_tag, GRPC_ERROR_NONE,
|
1380
|
-
[](void*, grpc_cq_completion* completion) { gpr_free(completion); },
|
1381
|
-
nullptr,
|
1382
|
-
static_cast<grpc_cq_completion*>(
|
1383
|
-
gpr_malloc(sizeof(grpc_cq_completion))));
|
1384
|
-
} else {
|
1385
|
-
Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
|
1386
|
-
GRPC_ERROR_NONE);
|
1387
|
-
}
|
1423
|
+
EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
|
1388
1424
|
error = GRPC_CALL_OK;
|
1389
1425
|
goto done;
|
1390
1426
|
}
|
@@ -1400,7 +1436,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1400
1436
|
stream_op = &bctl->op_;
|
1401
1437
|
stream_op_payload = &stream_op_payload_;
|
1402
1438
|
|
1403
|
-
|
1439
|
+
// rewrite batch ops into a transport op
|
1404
1440
|
for (i = 0; i < nops; i++) {
|
1405
1441
|
op = &ops[i];
|
1406
1442
|
if (op->reserved != nullptr) {
|
@@ -1409,7 +1445,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1409
1445
|
}
|
1410
1446
|
switch (op->op) {
|
1411
1447
|
case GRPC_OP_SEND_INITIAL_METADATA: {
|
1412
|
-
|
1448
|
+
// Flag validation: currently allow no flags
|
1413
1449
|
if (!AreInitialMetadataFlagsValid(op->flags)) {
|
1414
1450
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1415
1451
|
goto done_with_error;
|
@@ -1422,7 +1458,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1422
1458
|
// algorithm by setting the initial metadata with key of
|
1423
1459
|
// GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
|
1424
1460
|
// with the compression algorithm mapped from compression level.
|
1425
|
-
|
1461
|
+
// process compression level
|
1426
1462
|
grpc_compression_level effective_compression_level =
|
1427
1463
|
GRPC_COMPRESS_LEVEL_NONE;
|
1428
1464
|
bool level_set = false;
|
@@ -1432,7 +1468,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1432
1468
|
level_set = true;
|
1433
1469
|
} else {
|
1434
1470
|
const grpc_compression_options copts =
|
1435
|
-
|
1471
|
+
channel()->compression_options();
|
1436
1472
|
if (copts.default_level.is_set) {
|
1437
1473
|
level_set = true;
|
1438
1474
|
effective_compression_level = copts.default_level.level;
|
@@ -1462,18 +1498,25 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1462
1498
|
}
|
1463
1499
|
// Ignore any te metadata key value pairs specified.
|
1464
1500
|
send_initial_metadata_.Remove(TeMetadata());
|
1465
|
-
|
1501
|
+
// TODO(ctiller): just make these the same variable?
|
1466
1502
|
if (is_client() && send_deadline() != Timestamp::InfFuture()) {
|
1467
1503
|
send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
|
1468
1504
|
}
|
1505
|
+
if (is_client()) {
|
1506
|
+
send_initial_metadata_.Set(
|
1507
|
+
WaitForReady(),
|
1508
|
+
WaitForReady::ValueType{
|
1509
|
+
(op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
1510
|
+
(op->flags &
|
1511
|
+
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
1512
|
+
}
|
1469
1513
|
stream_op_payload->send_initial_metadata.send_initial_metadata =
|
1470
1514
|
&send_initial_metadata_;
|
1471
|
-
stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
|
1472
|
-
op->flags;
|
1473
1515
|
if (is_client()) {
|
1474
|
-
stream_op_payload->send_initial_metadata.peer_string =
|
1516
|
+
stream_op_payload->send_initial_metadata.peer_string =
|
1517
|
+
peer_string_atm_ptr();
|
1475
1518
|
}
|
1476
|
-
|
1519
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1477
1520
|
break;
|
1478
1521
|
}
|
1479
1522
|
case GRPC_OP_SEND_MESSAGE: {
|
@@ -1490,24 +1533,26 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1490
1533
|
goto done_with_error;
|
1491
1534
|
}
|
1492
1535
|
uint32_t flags = op->flags;
|
1493
|
-
|
1494
|
-
|
1495
|
-
|
1536
|
+
// If the outgoing buffer is already compressed, mark it as so in the
|
1537
|
+
// flags. These will be picked up by the compression filter and further
|
1538
|
+
// (wasteful) attempts at compression skipped.
|
1496
1539
|
if (op->data.send_message.send_message->data.raw.compression >
|
1497
1540
|
GRPC_COMPRESS_NONE) {
|
1498
1541
|
flags |= GRPC_WRITE_INTERNAL_COMPRESS;
|
1499
1542
|
}
|
1500
1543
|
stream_op->send_message = true;
|
1501
1544
|
sending_message_ = true;
|
1502
|
-
|
1503
|
-
|
1504
|
-
|
1505
|
-
|
1506
|
-
|
1545
|
+
send_slice_buffer_.Clear();
|
1546
|
+
grpc_slice_buffer_move_into(
|
1547
|
+
&op->data.send_message.send_message->data.raw.slice_buffer,
|
1548
|
+
send_slice_buffer_.c_slice_buffer());
|
1549
|
+
stream_op_payload->send_message.flags = flags;
|
1550
|
+
stream_op_payload->send_message.send_message = &send_slice_buffer_;
|
1551
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1507
1552
|
break;
|
1508
1553
|
}
|
1509
1554
|
case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
|
1510
|
-
|
1555
|
+
// Flag validation: currently allow no flags
|
1511
1556
|
if (op->flags != 0) {
|
1512
1557
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1513
1558
|
goto done_with_error;
|
@@ -1524,11 +1569,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1524
1569
|
sent_final_op_ = true;
|
1525
1570
|
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1526
1571
|
&send_trailing_metadata_;
|
1527
|
-
|
1572
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1528
1573
|
break;
|
1529
1574
|
}
|
1530
1575
|
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
1531
|
-
|
1576
|
+
// Flag validation: currently allow no flags
|
1532
1577
|
if (op->flags != 0) {
|
1533
1578
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1534
1579
|
goto done_with_error;
|
@@ -1558,11 +1603,10 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1558
1603
|
|
1559
1604
|
grpc_error_handle status_error =
|
1560
1605
|
op->data.send_status_from_server.status == GRPC_STATUS_OK
|
1561
|
-
?
|
1606
|
+
? absl::OkStatus()
|
1562
1607
|
: grpc_error_set_int(
|
1563
|
-
|
1564
|
-
|
1565
|
-
GRPC_ERROR_INT_GRPC_STATUS,
|
1608
|
+
GRPC_ERROR_CREATE("Server returned error"),
|
1609
|
+
StatusIntProperty::kRpcStatus,
|
1566
1610
|
static_cast<intptr_t>(
|
1567
1611
|
op->data.send_status_from_server.status));
|
1568
1612
|
if (op->data.send_status_from_server.status_details != nullptr) {
|
@@ -1570,16 +1614,15 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1570
1614
|
GrpcMessageMetadata(),
|
1571
1615
|
Slice(grpc_slice_copy(
|
1572
1616
|
*op->data.send_status_from_server.status_details)));
|
1573
|
-
if (status_error
|
1617
|
+
if (!status_error.ok()) {
|
1574
1618
|
status_error = grpc_error_set_str(
|
1575
|
-
status_error,
|
1619
|
+
status_error, StatusStrProperty::kGrpcMessage,
|
1576
1620
|
StringViewFromSlice(
|
1577
1621
|
*op->data.send_status_from_server.status_details));
|
1578
1622
|
}
|
1579
1623
|
}
|
1580
1624
|
|
1581
1625
|
status_error_.set(status_error);
|
1582
|
-
GRPC_ERROR_UNREF(status_error);
|
1583
1626
|
|
1584
1627
|
send_trailing_metadata_.Set(GrpcStatusMetadata(),
|
1585
1628
|
op->data.send_status_from_server.status);
|
@@ -1590,11 +1633,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1590
1633
|
&send_trailing_metadata_;
|
1591
1634
|
stream_op_payload->send_trailing_metadata.sent =
|
1592
1635
|
&sent_server_trailing_metadata_;
|
1593
|
-
|
1636
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1594
1637
|
break;
|
1595
1638
|
}
|
1596
1639
|
case GRPC_OP_RECV_INITIAL_METADATA: {
|
1597
|
-
|
1640
|
+
// Flag validation: currently allow no flags
|
1598
1641
|
if (op->flags != 0) {
|
1599
1642
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1600
1643
|
goto done_with_error;
|
@@ -1622,13 +1665,14 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1622
1665
|
stream_op_payload->recv_initial_metadata.trailing_metadata_available =
|
1623
1666
|
&is_trailers_only_;
|
1624
1667
|
} else {
|
1625
|
-
stream_op_payload->recv_initial_metadata.peer_string =
|
1668
|
+
stream_op_payload->recv_initial_metadata.peer_string =
|
1669
|
+
peer_string_atm_ptr();
|
1626
1670
|
}
|
1627
|
-
|
1671
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
|
1628
1672
|
break;
|
1629
1673
|
}
|
1630
1674
|
case GRPC_OP_RECV_MESSAGE: {
|
1631
|
-
|
1675
|
+
// Flag validation: currently allow no flags
|
1632
1676
|
if (op->flags != 0) {
|
1633
1677
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1634
1678
|
goto done_with_error;
|
@@ -1639,8 +1683,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1639
1683
|
}
|
1640
1684
|
receiving_message_ = true;
|
1641
1685
|
stream_op->recv_message = true;
|
1686
|
+
receiving_slice_buffer_.reset();
|
1642
1687
|
receiving_buffer_ = op->data.recv_message.recv_message;
|
1643
|
-
stream_op_payload->recv_message.recv_message = &
|
1688
|
+
stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
|
1689
|
+
receiving_stream_flags_ = 0;
|
1690
|
+
stream_op_payload->recv_message.flags = &receiving_stream_flags_;
|
1644
1691
|
stream_op_payload->recv_message.call_failed_before_recv_message =
|
1645
1692
|
&call_failed_before_recv_message_;
|
1646
1693
|
GRPC_CLOSURE_INIT(
|
@@ -1657,11 +1704,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1657
1704
|
bctl, grpc_schedule_on_exec_ctx);
|
1658
1705
|
stream_op_payload->recv_message.recv_message_ready =
|
1659
1706
|
&receiving_stream_ready_;
|
1660
|
-
|
1707
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
|
1661
1708
|
break;
|
1662
1709
|
}
|
1663
1710
|
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
1664
|
-
|
1711
|
+
// Flag validation: currently allow no flags
|
1665
1712
|
if (op->flags != 0) {
|
1666
1713
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1667
1714
|
goto done_with_error;
|
@@ -1696,11 +1743,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1696
1743
|
bctl, grpc_schedule_on_exec_ctx);
|
1697
1744
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1698
1745
|
&receiving_trailing_metadata_ready_;
|
1699
|
-
|
1746
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1700
1747
|
break;
|
1701
1748
|
}
|
1702
1749
|
case GRPC_OP_RECV_CLOSE_ON_SERVER: {
|
1703
|
-
|
1750
|
+
// Flag validation: currently allow no flags
|
1704
1751
|
if (op->flags != 0) {
|
1705
1752
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1706
1753
|
goto done_with_error;
|
@@ -1729,7 +1776,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1729
1776
|
bctl, grpc_schedule_on_exec_ctx);
|
1730
1777
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1731
1778
|
&receiving_trailing_metadata_ready_;
|
1732
|
-
|
1779
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1733
1780
|
break;
|
1734
1781
|
}
|
1735
1782
|
}
|
@@ -1739,9 +1786,9 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1739
1786
|
if (!is_notify_tag_closure) {
|
1740
1787
|
GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag));
|
1741
1788
|
}
|
1742
|
-
bctl->
|
1789
|
+
bctl->set_pending_ops(pending_ops);
|
1743
1790
|
|
1744
|
-
if (
|
1791
|
+
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
1745
1792
|
GRPC_CLOSURE_INIT(
|
1746
1793
|
&bctl->finish_batch_,
|
1747
1794
|
[](void* bctl, grpc_error_handle error) {
|
@@ -1751,24 +1798,26 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1751
1798
|
stream_op->on_complete = &bctl->finish_batch_;
|
1752
1799
|
}
|
1753
1800
|
|
1754
|
-
|
1801
|
+
if (grpc_call_trace.enabled()) {
|
1802
|
+
gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1803
|
+
PendingOpString(pending_ops).c_str(),
|
1804
|
+
grpc_transport_stream_op_batch_string(stream_op).c_str(),
|
1805
|
+
bctl->completion_data_.notify_tag.tag);
|
1806
|
+
}
|
1807
|
+
|
1755
1808
|
ExecuteBatch(stream_op, &bctl->start_batch_);
|
1756
1809
|
|
1757
1810
|
done:
|
1758
1811
|
return error;
|
1759
1812
|
|
1760
1813
|
done_with_error:
|
1761
|
-
|
1814
|
+
// reverse any mutations that occurred
|
1762
1815
|
if (stream_op->send_initial_metadata) {
|
1763
1816
|
sent_initial_metadata_ = false;
|
1764
1817
|
send_initial_metadata_.Clear();
|
1765
1818
|
}
|
1766
1819
|
if (stream_op->send_message) {
|
1767
1820
|
sending_message_ = false;
|
1768
|
-
// No need to invoke call->sending_stream->Orphan() explicitly.
|
1769
|
-
// stream_op_payload->send_message.send_message.reset() calls Deletor
|
1770
|
-
// of call->sending_stream which in-turn invokes the Orphan() method.
|
1771
|
-
stream_op_payload->send_message.send_message.reset();
|
1772
1821
|
}
|
1773
1822
|
if (stream_op->send_trailing_metadata) {
|
1774
1823
|
sent_final_op_ = false;
|
@@ -1795,8 +1844,1689 @@ void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
|
|
1795
1844
|
context_[elem].destroy = destroy;
|
1796
1845
|
}
|
1797
1846
|
|
1847
|
+
///////////////////////////////////////////////////////////////////////////////
|
1848
|
+
// Metadata validation helpers
|
1849
|
+
|
1850
|
+
namespace {
|
1851
|
+
bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
|
1852
|
+
for (size_t i = 0; i < count; i++) {
|
1853
|
+
grpc_metadata* md = &metadata[i];
|
1854
|
+
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
1855
|
+
grpc_validate_header_key_is_legal(md->key))) {
|
1856
|
+
return false;
|
1857
|
+
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1858
|
+
!GRPC_LOG_IF_ERROR(
|
1859
|
+
"validate_metadata",
|
1860
|
+
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1861
|
+
return false;
|
1862
|
+
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1863
|
+
// HTTP2 hpack encoding has a maximum limit.
|
1864
|
+
return false;
|
1865
|
+
}
|
1866
|
+
}
|
1867
|
+
return true;
|
1868
|
+
}
|
1869
|
+
} // namespace
|
1870
|
+
|
1871
|
+
///////////////////////////////////////////////////////////////////////////////
|
1872
|
+
// PromiseBasedCall
|
1873
|
+
// Will be folded into Call once the promise conversion is done
|
1874
|
+
|
1875
|
+
class PromiseBasedCall : public Call,
|
1876
|
+
public Activity,
|
1877
|
+
public Wakeable,
|
1878
|
+
public grpc_event_engine::experimental::EventEngine::
|
1879
|
+
Closure /* for deadlines */ {
|
1880
|
+
public:
|
1881
|
+
PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
1882
|
+
const grpc_call_create_args& args);
|
1883
|
+
|
1884
|
+
void ContextSet(grpc_context_index elem, void* value,
|
1885
|
+
void (*destroy)(void* value)) override;
|
1886
|
+
void* ContextGet(grpc_context_index elem) const override;
|
1887
|
+
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
1888
|
+
void SetCompletionQueueLocked(grpc_completion_queue* cq)
|
1889
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
1890
|
+
void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) {
|
1891
|
+
MutexLock lock(&mu_);
|
1892
|
+
CancelWithErrorLocked(std::move(error));
|
1893
|
+
}
|
1894
|
+
virtual void CancelWithErrorLocked(absl::Status error)
|
1895
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
1896
|
+
bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) {
|
1897
|
+
MutexLock lock(&mu_);
|
1898
|
+
return completed_;
|
1899
|
+
}
|
1900
|
+
|
1901
|
+
void Orphan() final {
|
1902
|
+
MutexLock lock(&mu_);
|
1903
|
+
if (!completed_) {
|
1904
|
+
CancelWithErrorLocked(absl::CancelledError("Call orphaned"));
|
1905
|
+
}
|
1906
|
+
}
|
1907
|
+
|
1908
|
+
// Implementation of call refcounting: move this to DualRefCounted once we
|
1909
|
+
// don't need to maintain FilterStackCall compatibility
|
1910
|
+
void ExternalRef() final {
|
1911
|
+
const uint64_t prev_ref_pair =
|
1912
|
+
refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed);
|
1913
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1914
|
+
gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(),
|
1915
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1916
|
+
GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair));
|
1917
|
+
}
|
1918
|
+
}
|
1919
|
+
void ExternalUnref() final {
|
1920
|
+
const uint64_t prev_ref_pair =
|
1921
|
+
refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel);
|
1922
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1923
|
+
gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(),
|
1924
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1925
|
+
GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1);
|
1926
|
+
}
|
1927
|
+
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
1928
|
+
if (GPR_UNLIKELY(strong_refs == 1)) {
|
1929
|
+
Orphan();
|
1930
|
+
}
|
1931
|
+
// Now drop the weak ref.
|
1932
|
+
InternalUnref("external_ref");
|
1933
|
+
}
|
1934
|
+
void InternalRef(const char* reason) final {
|
1935
|
+
uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed);
|
1936
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1937
|
+
gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason,
|
1938
|
+
GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n),
|
1939
|
+
GetWeakRefs(n) + 1);
|
1940
|
+
}
|
1941
|
+
}
|
1942
|
+
void InternalUnref(const char* reason) final {
|
1943
|
+
const uint64_t prev_ref_pair =
|
1944
|
+
refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel);
|
1945
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1946
|
+
gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(),
|
1947
|
+
reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1948
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1);
|
1949
|
+
}
|
1950
|
+
if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
|
1951
|
+
DeleteThis();
|
1952
|
+
}
|
1953
|
+
}
|
1954
|
+
|
1955
|
+
// Activity methods
|
1956
|
+
void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1957
|
+
Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override {
|
1958
|
+
InternalRef("wakeup");
|
1959
|
+
// If ASAN is defined, we leverage it to detect dropped Waker objects.
|
1960
|
+
// Usually Waker must be destroyed or woken up, but (especially with arenas)
|
1961
|
+
// it's not uncommon to create a Waker and then do neither. In that case it's
|
1962
|
+
// incredibly fraught to diagnose where the dropped reference to this object was
|
1963
|
+
// created. Instead, leverage ASAN and create a new object per expected wakeup.
|
1964
|
+
// Now when we drop such an object ASAN will fail and we'll get a callstack to
|
1965
|
+
// the creation of the waker in question.
|
1966
|
+
#if defined(__has_feature)
|
1967
|
+
#if __has_feature(address_sanitizer)
|
1968
|
+
#define GRPC_CALL_USES_ASAN_WAKER
|
1969
|
+
class AsanWaker final : public Wakeable {
|
1970
|
+
public:
|
1971
|
+
explicit AsanWaker(PromiseBasedCall* call) : call_(call) {}
|
1972
|
+
|
1973
|
+
void Wakeup(void*) override {
|
1974
|
+
call_->Wakeup(nullptr);
|
1975
|
+
delete this;
|
1976
|
+
}
|
1977
|
+
|
1978
|
+
void Drop(void*) override {
|
1979
|
+
call_->Drop(nullptr);
|
1980
|
+
delete this;
|
1981
|
+
}
|
1982
|
+
|
1983
|
+
std::string ActivityDebugTag(void*) const override {
|
1984
|
+
return call_->DebugTag();
|
1985
|
+
}
|
1986
|
+
|
1987
|
+
private:
|
1988
|
+
PromiseBasedCall* call_;
|
1989
|
+
};
|
1990
|
+
return Waker(new AsanWaker(this), nullptr);
|
1991
|
+
#endif
|
1992
|
+
#endif
|
1993
|
+
#ifndef GRPC_CALL_USES_ASAN_WAKER
|
1994
|
+
return Waker(this, nullptr);
|
1995
|
+
#endif
|
1996
|
+
}
|
1997
|
+
Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1998
|
+
|
1999
|
+
// Wakeable methods
|
2000
|
+
void Wakeup(void*) override {
|
2001
|
+
channel()->event_engine()->Run([this] {
|
2002
|
+
ApplicationCallbackExecCtx app_exec_ctx;
|
2003
|
+
ExecCtx exec_ctx;
|
2004
|
+
{
|
2005
|
+
ScopedContext activity_context(this);
|
2006
|
+
MutexLock lock(&mu_);
|
2007
|
+
Update();
|
2008
|
+
}
|
2009
|
+
InternalUnref("wakeup");
|
2010
|
+
});
|
2011
|
+
}
|
2012
|
+
void Drop(void*) override { InternalUnref("wakeup"); }
|
2013
|
+
|
2014
|
+
void RunInContext(absl::AnyInvocable<void()> fn) {
|
2015
|
+
if (Activity::current() == this) {
|
2016
|
+
fn();
|
2017
|
+
} else {
|
2018
|
+
InternalRef("in_context");
|
2019
|
+
channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable {
|
2020
|
+
ApplicationCallbackExecCtx app_exec_ctx;
|
2021
|
+
ExecCtx exec_ctx;
|
2022
|
+
{
|
2023
|
+
ScopedContext activity_context(this);
|
2024
|
+
MutexLock lock(&mu_);
|
2025
|
+
fn();
|
2026
|
+
Update();
|
2027
|
+
}
|
2028
|
+
InternalUnref("in_context");
|
2029
|
+
});
|
2030
|
+
}
|
2031
|
+
}
|
2032
|
+
|
2033
|
+
grpc_compression_algorithm test_only_compression_algorithm() override {
|
2034
|
+
abort();
|
2035
|
+
}
|
2036
|
+
uint32_t test_only_message_flags() override { abort(); }
|
2037
|
+
uint32_t test_only_encodings_accepted_by_peer() override { abort(); }
|
2038
|
+
grpc_compression_algorithm compression_for_level(
|
2039
|
+
grpc_compression_level) override {
|
2040
|
+
abort();
|
2041
|
+
}
|
2042
|
+
|
2043
|
+
// This should return nullptr for the promise stack (and alternative means
|
2044
|
+
// for that functionality be invented)
|
2045
|
+
grpc_call_stack* call_stack() override { return nullptr; }
|
2046
|
+
|
2047
|
+
void UpdateDeadline(Timestamp deadline);
|
2048
|
+
void ResetDeadline();
|
2049
|
+
|
2050
|
+
// Implementation of EventEngine::Closure, called when deadline expires
|
2051
|
+
void Run() override;
|
2052
|
+
|
2053
|
+
virtual ServerCallContext* server_call_context() { return nullptr; }
|
2054
|
+
|
2055
|
+
protected:
|
2056
|
+
class ScopedContext
|
2057
|
+
: public ScopedActivity,
|
2058
|
+
public promise_detail::Context<Arena>,
|
2059
|
+
public promise_detail::Context<grpc_call_context_element>,
|
2060
|
+
public promise_detail::Context<CallContext>,
|
2061
|
+
public promise_detail::Context<CallFinalization> {
|
2062
|
+
public:
|
2063
|
+
explicit ScopedContext(PromiseBasedCall* call)
|
2064
|
+
: ScopedActivity(call),
|
2065
|
+
promise_detail::Context<Arena>(call->arena()),
|
2066
|
+
promise_detail::Context<grpc_call_context_element>(call->context_),
|
2067
|
+
promise_detail::Context<CallContext>(&call->call_context_),
|
2068
|
+
promise_detail::Context<CallFinalization>(&call->finalization_) {}
|
2069
|
+
};
|
2070
|
+
|
2071
|
+
class Completion {
|
2072
|
+
public:
|
2073
|
+
Completion() : index_(kNullIndex) {}
|
2074
|
+
~Completion() { GPR_ASSERT(index_ == kNullIndex); }
|
2075
|
+
explicit Completion(uint8_t index) : index_(index) {}
|
2076
|
+
Completion(const Completion& other) = delete;
|
2077
|
+
Completion& operator=(const Completion& other) = delete;
|
2078
|
+
Completion(Completion&& other) noexcept : index_(other.index_) {
|
2079
|
+
other.index_ = kNullIndex;
|
2080
|
+
}
|
2081
|
+
Completion& operator=(Completion&& other) noexcept {
|
2082
|
+
GPR_ASSERT(index_ == kNullIndex);
|
2083
|
+
index_ = other.index_;
|
2084
|
+
other.index_ = kNullIndex;
|
2085
|
+
return *this;
|
2086
|
+
}
|
2087
|
+
|
2088
|
+
uint8_t index() const { return index_; }
|
2089
|
+
uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
|
2090
|
+
bool has_value() const { return index_ != kNullIndex; }
|
2091
|
+
|
2092
|
+
private:
|
2093
|
+
enum : uint8_t { kNullIndex = 0xff };
|
2094
|
+
uint8_t index_;
|
2095
|
+
};
|
2096
|
+
|
2097
|
+
~PromiseBasedCall() override {
|
2098
|
+
if (non_owning_wakeable_) non_owning_wakeable_->DropActivity();
|
2099
|
+
if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
|
2100
|
+
}
|
2101
|
+
|
2102
|
+
// Enumerates why a Completion is still pending
|
2103
|
+
enum class PendingOp {
|
2104
|
+
// We're in the midst of starting a batch of operations
|
2105
|
+
kStartingBatch = 0,
|
2106
|
+
// The following correspond with the batch operations from above
|
2107
|
+
kReceiveInitialMetadata,
|
2108
|
+
kReceiveStatusOnClient,
|
2109
|
+
kReceiveCloseOnServer = kReceiveStatusOnClient,
|
2110
|
+
kSendMessage,
|
2111
|
+
kReceiveMessage,
|
2112
|
+
kSendStatusFromServer,
|
2113
|
+
kSendCloseFromClient = kSendStatusFromServer,
|
2114
|
+
};
|
2115
|
+
|
2116
|
+
const char* PendingOpString(PendingOp reason) const {
|
2117
|
+
switch (reason) {
|
2118
|
+
case PendingOp::kStartingBatch:
|
2119
|
+
return "StartingBatch";
|
2120
|
+
case PendingOp::kReceiveInitialMetadata:
|
2121
|
+
return "ReceiveInitialMetadata";
|
2122
|
+
case PendingOp::kReceiveStatusOnClient:
|
2123
|
+
return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
|
2124
|
+
case PendingOp::kSendMessage:
|
2125
|
+
return "SendMessage";
|
2126
|
+
case PendingOp::kReceiveMessage:
|
2127
|
+
return "ReceiveMessage";
|
2128
|
+
case PendingOp::kSendStatusFromServer:
|
2129
|
+
return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
|
2130
|
+
}
|
2131
|
+
return "Unknown";
|
2132
|
+
}
|
2133
|
+
|
2134
|
+
static constexpr uint8_t PendingOpBit(PendingOp reason) {
|
2135
|
+
return 1 << static_cast<int>(reason);
|
2136
|
+
}
|
2137
|
+
|
2138
|
+
Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; }
|
2139
|
+
// Begin work on a completion, recording the tag/closure to notify.
|
2140
|
+
// Use the op selected in \a ops to determine the index to allocate into.
|
2141
|
+
// Starts the "StartingBatch" PendingOp immediately.
|
2142
|
+
// Assumes at least one operation in \a ops.
|
2143
|
+
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops)
|
2144
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2145
|
+
// Add one pending op to the completion, and return it.
|
2146
|
+
Completion AddOpToCompletion(const Completion& completion, PendingOp reason)
|
2147
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2148
|
+
// Stringify a completion
|
2149
|
+
std::string CompletionString(const Completion& completion) const {
|
2150
|
+
return completion.has_value()
|
2151
|
+
? absl::StrFormat(
|
2152
|
+
"%d:tag=%p", static_cast<int>(completion.index()),
|
2153
|
+
completion_info_[completion.index()].pending.tag)
|
2154
|
+
: "no-completion";
|
2155
|
+
}
|
2156
|
+
// Finish one op on the completion. Must have been previously been added.
|
2157
|
+
// The completion as a whole finishes when all pending ops finish.
|
2158
|
+
void FinishOpOnCompletion(Completion* completion, PendingOp reason)
|
2159
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2160
|
+
// Mark the completion as failed. Does not finish it.
|
2161
|
+
void FailCompletion(const Completion& completion,
|
2162
|
+
SourceLocation source_location = {});
|
2163
|
+
// Run the promise polling loop until it stalls.
|
2164
|
+
void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2165
|
+
// Update the promise state once.
|
2166
|
+
virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
2167
|
+
// Accept the stats from the context (call once we have proof the transport is
|
2168
|
+
// done with them).
|
2169
|
+
// Right now this means that promise based calls do not record correct stats
|
2170
|
+
// with census if they are cancelled.
|
2171
|
+
// TODO(ctiller): this should be remedied before promise based calls are
|
2172
|
+
// dexperimentalized.
|
2173
|
+
void AcceptTransportStatsFromContext() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2174
|
+
final_stats_ = *call_context_.call_stats();
|
2175
|
+
}
|
2176
|
+
|
2177
|
+
grpc_completion_queue* cq() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return cq_; }
|
2178
|
+
|
2179
|
+
void CToMetadata(grpc_metadata* metadata, size_t count,
|
2180
|
+
grpc_metadata_batch* batch);
|
2181
|
+
|
2182
|
+
std::string ActivityDebugTag(void*) const override { return DebugTag(); }
|
2183
|
+
|
2184
|
+
// At the end of the call run any finalization actions.
|
2185
|
+
void RunFinalization(grpc_status_code status, const char* status_details) {
|
2186
|
+
grpc_call_final_info final_info;
|
2187
|
+
final_info.stats = final_stats_;
|
2188
|
+
final_info.final_status = status;
|
2189
|
+
final_info.error_string = status_details;
|
2190
|
+
finalization_.Run(&final_info);
|
2191
|
+
}
|
2192
|
+
|
2193
|
+
std::string PresentAndCompletionText(const char* caption, bool has,
|
2194
|
+
const Completion& completion) const {
|
2195
|
+
if (has) {
|
2196
|
+
if (completion.has_value()) {
|
2197
|
+
return absl::StrCat(caption, ":", CompletionString(completion), " ");
|
2198
|
+
} else {
|
2199
|
+
return absl::StrCat(caption,
|
2200
|
+
":!!BUG:operation is present, no completion!! ");
|
2201
|
+
}
|
2202
|
+
} else {
|
2203
|
+
if (!completion.has_value()) {
|
2204
|
+
return "";
|
2205
|
+
} else {
|
2206
|
+
return absl::StrCat(caption, ":no-op:", CompletionString(completion),
|
2207
|
+
" ");
|
2208
|
+
}
|
2209
|
+
}
|
2210
|
+
}
|
2211
|
+
|
2212
|
+
std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2213
|
+
return absl::StrCat(PresentAndCompletionText("outstanding_send",
|
2214
|
+
outstanding_send_.has_value(),
|
2215
|
+
send_message_completion_)
|
2216
|
+
.c_str(),
|
2217
|
+
PresentAndCompletionText("outstanding_recv",
|
2218
|
+
outstanding_recv_.has_value(),
|
2219
|
+
recv_message_completion_)
|
2220
|
+
.c_str());
|
2221
|
+
}
|
2222
|
+
|
2223
|
+
void StartRecvMessage(const grpc_op& op, const Completion& completion,
|
2224
|
+
PipeReceiver<MessageHandle>* receiver)
|
2225
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2226
|
+
void PollRecvMessage(grpc_compression_algorithm compression_algorithm)
|
2227
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2228
|
+
void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2229
|
+
void StartSendMessage(const grpc_op& op, const Completion& completion,
|
2230
|
+
PipeSender<MessageHandle>* sender)
|
2231
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2232
|
+
bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2233
|
+
void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2234
|
+
|
2235
|
+
bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2236
|
+
return completed_;
|
2237
|
+
}
|
2238
|
+
void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; }
|
2239
|
+
bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2240
|
+
return outstanding_send_.has_value();
|
2241
|
+
}
|
2242
|
+
|
2243
|
+
private:
|
2244
|
+
union CompletionInfo {
|
2245
|
+
struct Pending {
|
2246
|
+
// Bitmask of PendingOps
|
2247
|
+
uint8_t pending_op_bits;
|
2248
|
+
bool is_closure;
|
2249
|
+
bool success;
|
2250
|
+
void* tag;
|
2251
|
+
} pending;
|
2252
|
+
grpc_cq_completion completion;
|
2253
|
+
};
|
2254
|
+
|
2255
|
+
class NonOwningWakable final : public Wakeable {
|
2256
|
+
public:
|
2257
|
+
explicit NonOwningWakable(PromiseBasedCall* call) : call_(call) {}
|
2258
|
+
|
2259
|
+
// Ref the Handle (not the activity).
|
2260
|
+
void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
|
2261
|
+
|
2262
|
+
// Activity is going away... drop its reference and sever the connection
|
2263
|
+
// back.
|
2264
|
+
void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
|
2265
|
+
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2266
|
+
MutexLock lock(&mu_);
|
2267
|
+
GPR_ASSERT(call_ != nullptr);
|
2268
|
+
call_ = nullptr;
|
2269
|
+
}
|
2270
|
+
|
2271
|
+
// Activity needs to wake up (if it still exists!) - wake it up, and drop
|
2272
|
+
// the ref that was kept for this handle.
|
2273
|
+
void Wakeup(void*) override ABSL_LOCKS_EXCLUDED(mu_) {
|
2274
|
+
// Drop the ref to the handle at end of scope (we have one ref = one
|
2275
|
+
// wakeup semantics).
|
2276
|
+
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2277
|
+
ReleasableMutexLock lock(&mu_);
|
2278
|
+
// Note that activity refcount can drop to zero, but we could win the lock
|
2279
|
+
// against DropActivity, so we need to only increase activities refcount
|
2280
|
+
// if it is non-zero.
|
2281
|
+
PromiseBasedCall* call = call_;
|
2282
|
+
if (call != nullptr && call->RefIfNonZero()) {
|
2283
|
+
lock.Release();
|
2284
|
+
// Activity still exists and we have a reference: wake it up, which will
|
2285
|
+
// drop the ref.
|
2286
|
+
call->Wakeup(nullptr);
|
2287
|
+
}
|
2288
|
+
}
|
2289
|
+
|
2290
|
+
std::string ActivityDebugTag(void*) const override {
|
2291
|
+
MutexLock lock(&mu_);
|
2292
|
+
return call_ == nullptr ? "<unknown>" : call_->DebugTag();
|
2293
|
+
}
|
2294
|
+
|
2295
|
+
void Drop(void*) override { Unref(); }
|
2296
|
+
|
2297
|
+
private:
|
2298
|
+
// Unref the Handle (not the activity).
|
2299
|
+
void Unref() {
|
2300
|
+
if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) {
|
2301
|
+
delete this;
|
2302
|
+
}
|
2303
|
+
}
|
2304
|
+
|
2305
|
+
mutable Mutex mu_;
|
2306
|
+
// We have two initial refs: one for the wakeup that this is created for,
|
2307
|
+
// and will be dropped by Wakeup, and the other for the activity which is
|
2308
|
+
// dropped by DropActivity.
|
2309
|
+
std::atomic<size_t> refs_{2};
|
2310
|
+
PromiseBasedCall* call_ ABSL_GUARDED_BY(mu_);
|
2311
|
+
};
|
2312
|
+
|
2313
|
+
static void OnDestroy(void* arg, grpc_error_handle) {
|
2314
|
+
auto* call = static_cast<PromiseBasedCall*>(arg);
|
2315
|
+
ScopedContext context(call);
|
2316
|
+
call->DeleteThis();
|
2317
|
+
}
|
2318
|
+
|
2319
|
+
// First 32 bits are strong refs, next 32 bits are weak refs.
|
2320
|
+
static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
|
2321
|
+
return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
|
2322
|
+
}
|
2323
|
+
static uint32_t GetStrongRefs(uint64_t ref_pair) {
|
2324
|
+
return static_cast<uint32_t>(ref_pair >> 32);
|
2325
|
+
}
|
2326
|
+
static uint32_t GetWeakRefs(uint64_t ref_pair) {
|
2327
|
+
return static_cast<uint32_t>(ref_pair & 0xffffffffu);
|
2328
|
+
}
|
2329
|
+
|
2330
|
+
bool RefIfNonZero() {
|
2331
|
+
uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire);
|
2332
|
+
do {
|
2333
|
+
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
2334
|
+
if (strong_refs == 0) return false;
|
2335
|
+
} while (!refs_.compare_exchange_weak(
|
2336
|
+
prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0),
|
2337
|
+
std::memory_order_acq_rel, std::memory_order_acquire));
|
2338
|
+
return true;
|
2339
|
+
}
|
2340
|
+
|
2341
|
+
mutable Mutex mu_;
|
2342
|
+
std::atomic<uint64_t> refs_;
|
2343
|
+
CallContext call_context_{this};
|
2344
|
+
bool keep_polling_ ABSL_GUARDED_BY(mu()) = false;
|
2345
|
+
|
2346
|
+
// Contexts for various subsystems (security, tracing, ...).
|
2347
|
+
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
2348
|
+
grpc_completion_queue* cq_ ABSL_GUARDED_BY(mu_);
|
2349
|
+
NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr;
|
2350
|
+
CompletionInfo completion_info_[6];
|
2351
|
+
grpc_call_stats final_stats_{};
|
2352
|
+
CallFinalization finalization_;
|
2353
|
+
// Current deadline.
|
2354
|
+
Timestamp deadline_ = Timestamp::InfFuture();
|
2355
|
+
grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_;
|
2356
|
+
absl::optional<PipeSender<MessageHandle>::PushType> outstanding_send_
|
2357
|
+
ABSL_GUARDED_BY(mu_);
|
2358
|
+
absl::optional<PipeReceiverNextType<MessageHandle>> outstanding_recv_
|
2359
|
+
ABSL_GUARDED_BY(mu_);
|
2360
|
+
grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr;
|
2361
|
+
Completion send_message_completion_ ABSL_GUARDED_BY(mu_);
|
2362
|
+
Completion recv_message_completion_ ABSL_GUARDED_BY(mu_);
|
2363
|
+
bool completed_ ABSL_GUARDED_BY(mu_) = false;
|
2364
|
+
};
|
2365
|
+
|
2366
|
+
template <typename T>
|
2367
|
+
grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
|
2368
|
+
grpc_call** out_call) {
|
2369
|
+
Channel* channel = args->channel.get();
|
2370
|
+
|
2371
|
+
auto alloc = Arena::CreateWithAlloc(channel->CallSizeEstimate(), sizeof(T),
|
2372
|
+
channel->allocator());
|
2373
|
+
PromiseBasedCall* call = new (alloc.second) T(alloc.first, args);
|
2374
|
+
*out_call = call->c_ptr();
|
2375
|
+
GPR_DEBUG_ASSERT(Call::FromC(*out_call) == call);
|
2376
|
+
return absl::OkStatus();
|
2377
|
+
}
|
2378
|
+
|
2379
|
+
PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2380
|
+
const grpc_call_create_args& args)
|
2381
|
+
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
2382
|
+
args.channel->Ref()),
|
2383
|
+
refs_(MakeRefPair(initial_external_refs, 0)),
|
2384
|
+
cq_(args.cq) {
|
2385
|
+
if (args.cq != nullptr) {
|
2386
|
+
GPR_ASSERT(args.pollset_set_alternative == nullptr &&
|
2387
|
+
"Only one of 'cq' and 'pollset_set_alternative' should be "
|
2388
|
+
"non-nullptr.");
|
2389
|
+
GRPC_CQ_INTERNAL_REF(args.cq, "bind");
|
2390
|
+
call_context_.pollent_ =
|
2391
|
+
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args.cq));
|
2392
|
+
}
|
2393
|
+
if (args.pollset_set_alternative != nullptr) {
|
2394
|
+
call_context_.pollent_ = grpc_polling_entity_create_from_pollset_set(
|
2395
|
+
args.pollset_set_alternative);
|
2396
|
+
}
|
2397
|
+
}
|
2398
|
+
|
2399
|
+
Waker PromiseBasedCall::MakeNonOwningWaker() {
|
2400
|
+
if (non_owning_wakeable_ == nullptr) {
|
2401
|
+
non_owning_wakeable_ = new NonOwningWakable(this);
|
2402
|
+
} else {
|
2403
|
+
non_owning_wakeable_->Ref();
|
2404
|
+
}
|
2405
|
+
return Waker(non_owning_wakeable_, nullptr);
|
2406
|
+
}
|
2407
|
+
|
2408
|
+
void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count,
|
2409
|
+
grpc_metadata_batch* b) {
|
2410
|
+
for (size_t i = 0; i < count; i++) {
|
2411
|
+
grpc_metadata* md = &metadata[i];
|
2412
|
+
auto key = StringViewFromSlice(md->key);
|
2413
|
+
// Filter "content-length metadata"
|
2414
|
+
if (key == "content-length") continue;
|
2415
|
+
b->Append(key, Slice(CSliceRef(md->value)),
|
2416
|
+
[md](absl::string_view error, const Slice& value) {
|
2417
|
+
gpr_log(GPR_DEBUG, "Append error: %s",
|
2418
|
+
absl::StrCat("key=", StringViewFromSlice(md->key),
|
2419
|
+
" error=", error,
|
2420
|
+
" value=", value.as_string_view())
|
2421
|
+
.c_str());
|
2422
|
+
});
|
2423
|
+
}
|
2424
|
+
}
|
2425
|
+
|
2426
|
+
void PromiseBasedCall::ContextSet(grpc_context_index elem, void* value,
|
2427
|
+
void (*destroy)(void*)) {
|
2428
|
+
if (context_[elem].destroy != nullptr) {
|
2429
|
+
context_[elem].destroy(context_[elem].value);
|
2430
|
+
}
|
2431
|
+
context_[elem].value = value;
|
2432
|
+
context_[elem].destroy = destroy;
|
2433
|
+
}
|
2434
|
+
|
2435
|
+
void* PromiseBasedCall::ContextGet(grpc_context_index elem) const {
|
2436
|
+
return context_[elem].value;
|
2437
|
+
}
|
2438
|
+
|
2439
|
+
PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
|
2440
|
+
void* tag, bool is_closure, const grpc_op* ops) {
|
2441
|
+
Completion c(BatchSlotForOp(ops[0].op));
|
2442
|
+
if (grpc_call_trace.enabled()) {
|
2443
|
+
gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(),
|
2444
|
+
CompletionString(c).c_str(), tag);
|
2445
|
+
}
|
2446
|
+
if (!is_closure) {
|
2447
|
+
grpc_cq_begin_op(cq(), tag);
|
2448
|
+
}
|
2449
|
+
completion_info_[c.index()].pending = {
|
2450
|
+
PendingOpBit(PendingOp::kStartingBatch), is_closure, true, tag};
|
2451
|
+
return c;
|
2452
|
+
}
|
2453
|
+
|
2454
|
+
PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
|
2455
|
+
const Completion& completion, PendingOp reason) {
|
2456
|
+
if (grpc_call_trace.enabled()) {
|
2457
|
+
gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
|
2458
|
+
CompletionString(completion).c_str(), PendingOpString(reason));
|
2459
|
+
}
|
2460
|
+
GPR_ASSERT(completion.has_value());
|
2461
|
+
auto& pending_op_bits =
|
2462
|
+
completion_info_[completion.index()].pending.pending_op_bits;
|
2463
|
+
GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0);
|
2464
|
+
pending_op_bits |= PendingOpBit(reason);
|
2465
|
+
return Completion(completion.index());
|
2466
|
+
}
|
2467
|
+
|
2468
|
+
void PromiseBasedCall::FailCompletion(const Completion& completion,
|
2469
|
+
SourceLocation location) {
|
2470
|
+
if (grpc_call_trace.enabled()) {
|
2471
|
+
gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
|
2472
|
+
"%s[call] FailCompletion %s", DebugTag().c_str(),
|
2473
|
+
CompletionString(completion).c_str());
|
2474
|
+
}
|
2475
|
+
completion_info_[completion.index()].pending.success = false;
|
2476
|
+
}
|
2477
|
+
|
2478
|
+
void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
|
2479
|
+
PendingOp reason) {
|
2480
|
+
if (grpc_call_trace.enabled()) {
|
2481
|
+
auto pending_op_bits =
|
2482
|
+
completion_info_[completion->index()].pending.pending_op_bits;
|
2483
|
+
bool success = completion_info_[completion->index()].pending.success;
|
2484
|
+
std::vector<const char*> pending;
|
2485
|
+
for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) {
|
2486
|
+
if (static_cast<PendingOp>(i) == reason) continue;
|
2487
|
+
if (pending_op_bits & (1 << i)) {
|
2488
|
+
pending.push_back(PendingOpString(static_cast<PendingOp>(i)));
|
2489
|
+
}
|
2490
|
+
}
|
2491
|
+
gpr_log(
|
2492
|
+
GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s",
|
2493
|
+
DebugTag().c_str(), completion_info_[completion->index()].pending.tag,
|
2494
|
+
CompletionString(*completion).c_str(), PendingOpString(reason),
|
2495
|
+
(pending.empty()
|
2496
|
+
? (success ? std::string("done") : std::string("failed"))
|
2497
|
+
: absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ",")))
|
2498
|
+
.c_str());
|
2499
|
+
}
|
2500
|
+
const uint8_t i = completion->TakeIndex();
|
2501
|
+
GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_));
|
2502
|
+
CompletionInfo::Pending& pending = completion_info_[i].pending;
|
2503
|
+
GPR_ASSERT(pending.pending_op_bits & PendingOpBit(reason));
|
2504
|
+
pending.pending_op_bits &= ~PendingOpBit(reason);
|
2505
|
+
auto error = pending.success ? absl::OkStatus() : absl::CancelledError();
|
2506
|
+
if (pending.pending_op_bits == 0) {
|
2507
|
+
if (pending.is_closure) {
|
2508
|
+
ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
|
2509
|
+
error);
|
2510
|
+
} else {
|
2511
|
+
grpc_cq_end_op(
|
2512
|
+
cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
|
2513
|
+
&completion_info_[i].completion);
|
2514
|
+
}
|
2515
|
+
}
|
2516
|
+
}
|
2517
|
+
|
2518
|
+
void PromiseBasedCall::Update() {
|
2519
|
+
keep_polling_ = false;
|
2520
|
+
do {
|
2521
|
+
UpdateOnce();
|
2522
|
+
} while (std::exchange(keep_polling_, false));
|
2523
|
+
}
|
2524
|
+
|
2525
|
+
void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; }
|
2526
|
+
|
2527
|
+
void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
2528
|
+
MutexLock lock(&mu_);
|
2529
|
+
SetCompletionQueueLocked(cq);
|
2530
|
+
}
|
2531
|
+
|
2532
|
+
void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
|
2533
|
+
cq_ = cq;
|
2534
|
+
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
2535
|
+
call_context_.pollent_ =
|
2536
|
+
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
|
2537
|
+
}
|
2538
|
+
|
2539
|
+
void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
|
2540
|
+
if (deadline >= deadline_) return;
|
2541
|
+
auto* const event_engine = channel()->event_engine();
|
2542
|
+
if (deadline_ != Timestamp::InfFuture()) {
|
2543
|
+
if (!event_engine->Cancel(deadline_task_)) return;
|
2544
|
+
} else {
|
2545
|
+
InternalRef("deadline");
|
2546
|
+
}
|
2547
|
+
event_engine->RunAfter(deadline - Timestamp::Now(), this);
|
2548
|
+
}
|
2549
|
+
|
2550
|
+
void PromiseBasedCall::ResetDeadline() {
|
2551
|
+
if (deadline_ == Timestamp::InfFuture()) return;
|
2552
|
+
auto* const event_engine = channel()->event_engine();
|
2553
|
+
if (!event_engine->Cancel(deadline_task_)) return;
|
2554
|
+
deadline_ = Timestamp::InfFuture();
|
2555
|
+
InternalUnref("deadline");
|
2556
|
+
}
|
2557
|
+
|
2558
|
+
void PromiseBasedCall::Run() {
|
2559
|
+
ApplicationCallbackExecCtx callback_exec_ctx;
|
2560
|
+
ExecCtx exec_ctx;
|
2561
|
+
CancelWithError(absl::DeadlineExceededError("Deadline exceeded"));
|
2562
|
+
InternalUnref("deadline");
|
2563
|
+
}
|
2564
|
+
|
2565
|
+
void PromiseBasedCall::StartSendMessage(const grpc_op& op,
|
2566
|
+
const Completion& completion,
|
2567
|
+
PipeSender<MessageHandle>* sender) {
|
2568
|
+
GPR_ASSERT(!outstanding_send_.has_value());
|
2569
|
+
if (!completed_) {
|
2570
|
+
send_message_completion_ =
|
2571
|
+
AddOpToCompletion(completion, PendingOp::kSendMessage);
|
2572
|
+
SliceBuffer send;
|
2573
|
+
grpc_slice_buffer_swap(
|
2574
|
+
&op.data.send_message.send_message->data.raw.slice_buffer,
|
2575
|
+
send.c_slice_buffer());
|
2576
|
+
outstanding_send_.emplace(sender->Push(
|
2577
|
+
GetContext<Arena>()->MakePooled<Message>(std::move(send), op.flags)));
|
2578
|
+
} else {
|
2579
|
+
FailCompletion(completion);
|
2580
|
+
}
|
2581
|
+
}
|
2582
|
+
|
2583
|
+
bool PromiseBasedCall::PollSendMessage() {
|
2584
|
+
if (!outstanding_send_.has_value()) return true;
|
2585
|
+
Poll<bool> r = (*outstanding_send_)();
|
2586
|
+
if (const bool* result = r.value_if_ready()) {
|
2587
|
+
if (grpc_call_trace.enabled()) {
|
2588
|
+
gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(),
|
2589
|
+
*result ? "successfully" : "with failure");
|
2590
|
+
}
|
2591
|
+
if (!*result) {
|
2592
|
+
FailCompletion(send_message_completion_);
|
2593
|
+
return false;
|
2594
|
+
}
|
2595
|
+
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2596
|
+
outstanding_send_.reset();
|
2597
|
+
}
|
2598
|
+
return true;
|
2599
|
+
}
|
2600
|
+
|
2601
|
+
void PromiseBasedCall::CancelSendMessage() {
|
2602
|
+
if (!outstanding_send_.has_value()) return;
|
2603
|
+
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2604
|
+
outstanding_send_.reset();
|
2605
|
+
}
|
2606
|
+
|
2607
|
+
void PromiseBasedCall::StartRecvMessage(const grpc_op& op,
|
2608
|
+
const Completion& completion,
|
2609
|
+
PipeReceiver<MessageHandle>* receiver) {
|
2610
|
+
GPR_ASSERT(!outstanding_recv_.has_value());
|
2611
|
+
recv_message_ = op.data.recv_message.recv_message;
|
2612
|
+
recv_message_completion_ =
|
2613
|
+
AddOpToCompletion(completion, PendingOp::kReceiveMessage);
|
2614
|
+
outstanding_recv_.emplace(receiver->Next());
|
2615
|
+
}
|
2616
|
+
|
2617
|
+
void PromiseBasedCall::PollRecvMessage(
|
2618
|
+
grpc_compression_algorithm incoming_compression_algorithm) {
|
2619
|
+
if (!outstanding_recv_.has_value()) return;
|
2620
|
+
Poll<NextResult<MessageHandle>> r = (*outstanding_recv_)();
|
2621
|
+
if (auto* result = r.value_if_ready()) {
|
2622
|
+
outstanding_recv_.reset();
|
2623
|
+
if (result->has_value()) {
|
2624
|
+
MessageHandle& message = **result;
|
2625
|
+
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
2626
|
+
(incoming_compression_algorithm != GRPC_COMPRESS_NONE)) {
|
2627
|
+
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
2628
|
+
nullptr, 0, incoming_compression_algorithm);
|
2629
|
+
} else {
|
2630
|
+
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
2631
|
+
}
|
2632
|
+
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
2633
|
+
&(*recv_message_)->data.raw.slice_buffer);
|
2634
|
+
if (grpc_call_trace.enabled()) {
|
2635
|
+
gpr_log(GPR_INFO,
|
2636
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2637
|
+
"%" PRIdPTR " byte message",
|
2638
|
+
DebugTag().c_str(),
|
2639
|
+
(*recv_message_)->data.raw.slice_buffer.length);
|
2640
|
+
}
|
2641
|
+
} else if (result->cancelled()) {
|
2642
|
+
if (grpc_call_trace.enabled()) {
|
2643
|
+
gpr_log(GPR_INFO,
|
2644
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2645
|
+
"end-of-stream with error",
|
2646
|
+
DebugTag().c_str());
|
2647
|
+
}
|
2648
|
+
FailCompletion(recv_message_completion_);
|
2649
|
+
*recv_message_ = nullptr;
|
2650
|
+
} else {
|
2651
|
+
if (grpc_call_trace.enabled()) {
|
2652
|
+
gpr_log(GPR_INFO,
|
2653
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2654
|
+
"end-of-stream",
|
2655
|
+
DebugTag().c_str());
|
2656
|
+
}
|
2657
|
+
*recv_message_ = nullptr;
|
2658
|
+
}
|
2659
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2660
|
+
} else if (completed_) {
|
2661
|
+
if (grpc_call_trace.enabled()) {
|
2662
|
+
gpr_log(GPR_INFO,
|
2663
|
+
"%s[call] UpdateOnce: outstanding_recv finishes: promise has "
|
2664
|
+
"completed without queuing a message, forcing end-of-stream",
|
2665
|
+
DebugTag().c_str());
|
2666
|
+
}
|
2667
|
+
outstanding_recv_.reset();
|
2668
|
+
*recv_message_ = nullptr;
|
2669
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2670
|
+
}
|
2671
|
+
}
|
2672
|
+
|
2673
|
+
void PromiseBasedCall::CancelRecvMessage() {
|
2674
|
+
if (!outstanding_recv_.has_value()) return;
|
2675
|
+
*recv_message_ = nullptr;
|
2676
|
+
outstanding_recv_.reset();
|
2677
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2678
|
+
}
|
2679
|
+
|
2680
|
+
///////////////////////////////////////////////////////////////////////////////
|
2681
|
+
// CallContext
|
2682
|
+
|
2683
|
+
void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
|
2684
|
+
call_->RunInContext(std::move(fn));
|
2685
|
+
}
|
2686
|
+
|
2687
|
+
void CallContext::IncrementRefCount(const char* reason) {
|
2688
|
+
call_->InternalRef(reason);
|
2689
|
+
}
|
2690
|
+
|
2691
|
+
void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
|
2692
|
+
|
2693
|
+
gpr_atm* CallContext::peer_string_atm_ptr() {
|
2694
|
+
return call_->peer_string_atm_ptr();
|
2695
|
+
}
|
2696
|
+
|
2697
|
+
void CallContext::UpdateDeadline(Timestamp deadline) {
|
2698
|
+
call_->UpdateDeadline(deadline);
|
2699
|
+
}
|
2700
|
+
|
2701
|
+
ServerCallContext* CallContext::server_call_context() {
|
2702
|
+
return call_->server_call_context();
|
2703
|
+
}
|
2704
|
+
|
2705
|
+
///////////////////////////////////////////////////////////////////////////////
|
2706
|
+
// PublishMetadataArray
|
2707
|
+
|
2708
|
+
namespace {
|
2709
|
+
void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) {
|
2710
|
+
const auto md_count = md->count();
|
2711
|
+
if (md_count > array->capacity) {
|
2712
|
+
array->capacity =
|
2713
|
+
std::max(array->capacity + md->count(), array->capacity * 3 / 2);
|
2714
|
+
array->metadata = static_cast<grpc_metadata*>(
|
2715
|
+
gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
|
2716
|
+
}
|
2717
|
+
PublishToAppEncoder encoder(array);
|
2718
|
+
md->Encode(&encoder);
|
2719
|
+
}
|
2720
|
+
} // namespace
|
2721
|
+
|
2722
|
+
///////////////////////////////////////////////////////////////////////////////
|
2723
|
+
// ClientPromiseBasedCall
|
2724
|
+
|
2725
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
2726
|
+
class ClientPromiseBasedCall final : public PromiseBasedCall {
|
2727
|
+
public:
|
2728
|
+
ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
|
2729
|
+
: PromiseBasedCall(arena, 1, *args) {
|
2730
|
+
global_stats().IncrementClientCallsCreated();
|
2731
|
+
ScopedContext context(this);
|
2732
|
+
send_initial_metadata_ =
|
2733
|
+
GetContext<Arena>()->MakePooled<ClientMetadata>(GetContext<Arena>());
|
2734
|
+
send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
|
2735
|
+
if (args->authority.has_value()) {
|
2736
|
+
send_initial_metadata_->Set(HttpAuthorityMetadata(),
|
2737
|
+
std::move(*args->authority));
|
2738
|
+
}
|
2739
|
+
if (auto* channelz_channel = channel()->channelz_node()) {
|
2740
|
+
channelz_channel->RecordCallStarted();
|
2741
|
+
}
|
2742
|
+
if (args->send_deadline != Timestamp::InfFuture()) {
|
2743
|
+
UpdateDeadline(args->send_deadline);
|
2744
|
+
}
|
2745
|
+
}
|
2746
|
+
|
2747
|
+
~ClientPromiseBasedCall() override {
|
2748
|
+
ScopedContext context(this);
|
2749
|
+
send_initial_metadata_.reset();
|
2750
|
+
recv_status_on_client_ = absl::monostate();
|
2751
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
2752
|
+
// Need to destroy the pipes under the ScopedContext above, so we move them
|
2753
|
+
// out here and then allow the destructors to run at end of scope, but
|
2754
|
+
// before context.
|
2755
|
+
auto c2s = std::move(client_to_server_messages_);
|
2756
|
+
auto s2c = std::move(server_to_client_messages_);
|
2757
|
+
auto sim = std::move(server_initial_metadata_);
|
2758
|
+
}
|
2759
|
+
|
2760
|
+
absl::string_view GetServerAuthority() const override { abort(); }
|
2761
|
+
void CancelWithErrorLocked(grpc_error_handle error) override
|
2762
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2763
|
+
bool is_trailers_only() const override {
|
2764
|
+
MutexLock lock(mu());
|
2765
|
+
return is_trailers_only_;
|
2766
|
+
}
|
2767
|
+
bool failed_before_recv_message() const override { abort(); }
|
2768
|
+
|
2769
|
+
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
2770
|
+
bool is_notify_tag_closure) override;
|
2771
|
+
|
2772
|
+
std::string DebugTag() const override {
|
2773
|
+
return absl::StrFormat("CLIENT_CALL[%p]: ", this);
|
2774
|
+
}
|
2775
|
+
|
2776
|
+
private:
|
2777
|
+
// Poll the underlying promise (and sundry objects) once.
|
2778
|
+
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
2779
|
+
// Finish the call with the given status/trailing metadata.
|
2780
|
+
void Finish(ServerMetadataHandle trailing_metadata)
|
2781
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2782
|
+
// Validate that a set of ops is valid for a client call.
|
2783
|
+
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const
|
2784
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2785
|
+
// Commit a valid batch of operations to be executed.
|
2786
|
+
void CommitBatch(const grpc_op* ops, size_t nops,
|
2787
|
+
const Completion& completion)
|
2788
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2789
|
+
// Start the underlying promise.
|
2790
|
+
void StartPromise(ClientMetadataHandle client_initial_metadata)
|
2791
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2792
|
+
// Publish status out to the application.
|
2793
|
+
void PublishStatus(
|
2794
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2795
|
+
ServerMetadataHandle trailing_metadata)
|
2796
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2797
|
+
// Publish server initial metadata out to the application.
|
2798
|
+
void PublishInitialMetadata(ServerMetadata* metadata)
|
2799
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2800
|
+
|
2801
|
+
ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
|
2802
|
+
Pipe<ServerMetadataHandle> server_initial_metadata_ ABSL_GUARDED_BY(mu()){
|
2803
|
+
arena()};
|
2804
|
+
Pipe<MessageHandle> client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2805
|
+
Pipe<MessageHandle> server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2806
|
+
|
2807
|
+
ClientMetadataHandle send_initial_metadata_;
|
2808
|
+
grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr;
|
2809
|
+
absl::variant<absl::monostate,
|
2810
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client,
|
2811
|
+
ServerMetadataHandle>
|
2812
|
+
recv_status_on_client_ ABSL_GUARDED_BY(mu());
|
2813
|
+
absl::optional<PipeReceiverNextType<ServerMetadataHandle>>
|
2814
|
+
server_initial_metadata_ready_;
|
2815
|
+
absl::optional<grpc_compression_algorithm> incoming_compression_algorithm_;
|
2816
|
+
Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu());
|
2817
|
+
Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu());
|
2818
|
+
Completion close_send_completion_ ABSL_GUARDED_BY(mu());
|
2819
|
+
bool is_trailers_only_ ABSL_GUARDED_BY(mu());
|
2820
|
+
};
|
2821
|
+
|
2822
|
+
void ClientPromiseBasedCall::StartPromise(
|
2823
|
+
ClientMetadataHandle client_initial_metadata) {
|
2824
|
+
GPR_ASSERT(!promise_.has_value());
|
2825
|
+
promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{
|
2826
|
+
std::move(client_initial_metadata),
|
2827
|
+
&server_initial_metadata_.sender,
|
2828
|
+
&client_to_server_messages_.receiver,
|
2829
|
+
&server_to_client_messages_.sender,
|
2830
|
+
});
|
2831
|
+
}
|
2832
|
+
|
2833
|
+
void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) {
|
2834
|
+
ScopedContext context(this);
|
2835
|
+
Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error)));
|
2836
|
+
}
|
2837
|
+
|
2838
|
+
grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
2839
|
+
size_t nops) const {
|
2840
|
+
BitSet<8> got_ops;
|
2841
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2842
|
+
const grpc_op& op = ops[op_idx];
|
2843
|
+
switch (op.op) {
|
2844
|
+
case GRPC_OP_SEND_INITIAL_METADATA:
|
2845
|
+
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
2846
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2847
|
+
}
|
2848
|
+
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
2849
|
+
op.data.send_initial_metadata.metadata)) {
|
2850
|
+
return GRPC_CALL_ERROR_INVALID_METADATA;
|
2851
|
+
}
|
2852
|
+
break;
|
2853
|
+
case GRPC_OP_SEND_MESSAGE:
|
2854
|
+
if (!AreWriteFlagsValid(op.flags)) {
|
2855
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2856
|
+
}
|
2857
|
+
break;
|
2858
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
2859
|
+
case GRPC_OP_RECV_MESSAGE:
|
2860
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2861
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
2862
|
+
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2863
|
+
break;
|
2864
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2865
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2866
|
+
return GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
2867
|
+
}
|
2868
|
+
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2869
|
+
got_ops.set(op.op);
|
2870
|
+
}
|
2871
|
+
return GRPC_CALL_OK;
|
2872
|
+
}
|
2873
|
+
|
2874
|
+
void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
2875
|
+
const Completion& completion) {
|
2876
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2877
|
+
const grpc_op& op = ops[op_idx];
|
2878
|
+
switch (op.op) {
|
2879
|
+
case GRPC_OP_SEND_INITIAL_METADATA: {
|
2880
|
+
// compression not implemented
|
2881
|
+
GPR_ASSERT(
|
2882
|
+
!op.data.send_initial_metadata.maybe_compression_level.is_set);
|
2883
|
+
if (!completed()) {
|
2884
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
2885
|
+
op.data.send_initial_metadata.count,
|
2886
|
+
send_initial_metadata_.get());
|
2887
|
+
StartPromise(std::move(send_initial_metadata_));
|
2888
|
+
}
|
2889
|
+
} break;
|
2890
|
+
case GRPC_OP_RECV_INITIAL_METADATA: {
|
2891
|
+
recv_initial_metadata_ =
|
2892
|
+
op.data.recv_initial_metadata.recv_initial_metadata;
|
2893
|
+
server_initial_metadata_ready_.emplace(
|
2894
|
+
server_initial_metadata_.receiver.Next());
|
2895
|
+
recv_initial_metadata_completion_ =
|
2896
|
+
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata);
|
2897
|
+
} break;
|
2898
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
2899
|
+
recv_status_on_client_completion_ =
|
2900
|
+
AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient);
|
2901
|
+
if (auto* finished_metadata =
|
2902
|
+
absl::get_if<ServerMetadataHandle>(&recv_status_on_client_)) {
|
2903
|
+
PublishStatus(op.data.recv_status_on_client,
|
2904
|
+
std::move(*finished_metadata));
|
2905
|
+
} else {
|
2906
|
+
recv_status_on_client_ = op.data.recv_status_on_client;
|
2907
|
+
}
|
2908
|
+
} break;
|
2909
|
+
case GRPC_OP_SEND_MESSAGE:
|
2910
|
+
StartSendMessage(op, completion, &client_to_server_messages_.sender);
|
2911
|
+
break;
|
2912
|
+
case GRPC_OP_RECV_MESSAGE:
|
2913
|
+
StartRecvMessage(op, completion, &server_to_client_messages_.receiver);
|
2914
|
+
break;
|
2915
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2916
|
+
close_send_completion_ =
|
2917
|
+
AddOpToCompletion(completion, PendingOp::kSendCloseFromClient);
|
2918
|
+
GPR_ASSERT(close_send_completion_.has_value());
|
2919
|
+
break;
|
2920
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2921
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2922
|
+
abort(); // unreachable
|
2923
|
+
}
|
2924
|
+
}
|
2925
|
+
}
|
2926
|
+
|
2927
|
+
grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
2928
|
+
size_t nops,
|
2929
|
+
void* notify_tag,
|
2930
|
+
bool is_notify_tag_closure) {
|
2931
|
+
MutexLock lock(mu());
|
2932
|
+
ScopedContext activity_context(this);
|
2933
|
+
if (nops == 0) {
|
2934
|
+
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
2935
|
+
return GRPC_CALL_OK;
|
2936
|
+
}
|
2937
|
+
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
2938
|
+
if (validation_result != GRPC_CALL_OK) {
|
2939
|
+
return validation_result;
|
2940
|
+
}
|
2941
|
+
Completion completion =
|
2942
|
+
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
2943
|
+
CommitBatch(ops, nops, completion);
|
2944
|
+
Update();
|
2945
|
+
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
2946
|
+
return GRPC_CALL_OK;
|
2947
|
+
}
|
2948
|
+
|
2949
|
+
void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) {
|
2950
|
+
incoming_compression_algorithm_ =
|
2951
|
+
metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
2952
|
+
server_initial_metadata_ready_.reset();
|
2953
|
+
GPR_ASSERT(recv_initial_metadata_ != nullptr);
|
2954
|
+
PublishMetadataArray(metadata,
|
2955
|
+
std::exchange(recv_initial_metadata_, nullptr));
|
2956
|
+
FinishOpOnCompletion(&recv_initial_metadata_completion_,
|
2957
|
+
PendingOp::kReceiveInitialMetadata);
|
2958
|
+
}
|
2959
|
+
|
2960
|
+
void ClientPromiseBasedCall::UpdateOnce() {
|
2961
|
+
if (grpc_call_trace.enabled()) {
|
2962
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s",
|
2963
|
+
DebugTag().c_str(),
|
2964
|
+
PresentAndCompletionText("server_initial_metadata_ready",
|
2965
|
+
server_initial_metadata_ready_.has_value(),
|
2966
|
+
recv_initial_metadata_completion_)
|
2967
|
+
.c_str(),
|
2968
|
+
PollStateDebugString().c_str(),
|
2969
|
+
promise_.has_value() ? "true" : "false");
|
2970
|
+
}
|
2971
|
+
if (server_initial_metadata_ready_.has_value()) {
|
2972
|
+
Poll<NextResult<ServerMetadataHandle>> r =
|
2973
|
+
(*server_initial_metadata_ready_)();
|
2974
|
+
if (auto* server_initial_metadata = r.value_if_ready()) {
|
2975
|
+
PublishInitialMetadata(server_initial_metadata->value().get());
|
2976
|
+
} else if (completed()) {
|
2977
|
+
ServerMetadata no_metadata{GetContext<Arena>()};
|
2978
|
+
PublishInitialMetadata(&no_metadata);
|
2979
|
+
}
|
2980
|
+
}
|
2981
|
+
if (!PollSendMessage()) {
|
2982
|
+
Finish(ServerMetadataFromStatus(absl::Status(
|
2983
|
+
absl::StatusCode::kInternal, "Failed to send message to server")));
|
2984
|
+
}
|
2985
|
+
if (!is_sending() && close_send_completion_.has_value()) {
|
2986
|
+
client_to_server_messages_.sender.Close();
|
2987
|
+
FinishOpOnCompletion(&close_send_completion_,
|
2988
|
+
PendingOp::kSendCloseFromClient);
|
2989
|
+
}
|
2990
|
+
if (promise_.has_value()) {
|
2991
|
+
Poll<ServerMetadataHandle> r = promise_();
|
2992
|
+
if (grpc_call_trace.enabled()) {
|
2993
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
|
2994
|
+
DebugTag().c_str(),
|
2995
|
+
PollToString(r, [](const ServerMetadataHandle& h) {
|
2996
|
+
return h->DebugString();
|
2997
|
+
}).c_str());
|
2998
|
+
}
|
2999
|
+
if (auto* result = r.value_if_ready()) {
|
3000
|
+
AcceptTransportStatsFromContext();
|
3001
|
+
Finish(std::move(*result));
|
3002
|
+
}
|
3003
|
+
}
|
3004
|
+
if (incoming_compression_algorithm_.has_value()) {
|
3005
|
+
PollRecvMessage(*incoming_compression_algorithm_);
|
3006
|
+
}
|
3007
|
+
}
|
3008
|
+
|
3009
|
+
void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
3010
|
+
if (grpc_call_trace.enabled()) {
|
3011
|
+
gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
|
3012
|
+
trailing_metadata->DebugString().c_str());
|
3013
|
+
}
|
3014
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3015
|
+
ResetDeadline();
|
3016
|
+
set_completed();
|
3017
|
+
if (recv_initial_metadata_ != nullptr) {
|
3018
|
+
ForceImmediateRepoll();
|
3019
|
+
}
|
3020
|
+
const bool pending_initial_metadata =
|
3021
|
+
server_initial_metadata_ready_.has_value();
|
3022
|
+
if (!pending_initial_metadata) {
|
3023
|
+
server_initial_metadata_ready_.emplace(
|
3024
|
+
server_initial_metadata_.receiver.Next());
|
3025
|
+
}
|
3026
|
+
Poll<NextResult<ServerMetadataHandle>> r =
|
3027
|
+
(*server_initial_metadata_ready_)();
|
3028
|
+
server_initial_metadata_ready_.reset();
|
3029
|
+
if (auto* result = r.value_if_ready()) {
|
3030
|
+
if (pending_initial_metadata) PublishInitialMetadata(result->value().get());
|
3031
|
+
is_trailers_only_ = false;
|
3032
|
+
} else {
|
3033
|
+
if (pending_initial_metadata) {
|
3034
|
+
ServerMetadata no_metadata{GetContext<Arena>()};
|
3035
|
+
PublishInitialMetadata(&no_metadata);
|
3036
|
+
}
|
3037
|
+
is_trailers_only_ = true;
|
3038
|
+
}
|
3039
|
+
if (auto* channelz_channel = channel()->channelz_node()) {
|
3040
|
+
if (trailing_metadata->get(GrpcStatusMetadata())
|
3041
|
+
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3042
|
+
channelz_channel->RecordCallSucceeded();
|
3043
|
+
} else {
|
3044
|
+
channelz_channel->RecordCallFailed();
|
3045
|
+
}
|
3046
|
+
}
|
3047
|
+
if (auto* status_request =
|
3048
|
+
absl::get_if<grpc_op::grpc_op_data::grpc_op_recv_status_on_client>(
|
3049
|
+
&recv_status_on_client_)) {
|
3050
|
+
PublishStatus(*status_request, std::move(trailing_metadata));
|
3051
|
+
} else {
|
3052
|
+
recv_status_on_client_ = std::move(trailing_metadata);
|
3053
|
+
}
|
3054
|
+
}
|
3055
|
+
|
3056
|
+
namespace {
|
3057
|
+
std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
|
3058
|
+
std::string out = absl::StrCat(
|
3059
|
+
trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
|
3060
|
+
? "Error received from peer"
|
3061
|
+
: "Error generated by client",
|
3062
|
+
"grpc_status: ",
|
3063
|
+
grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
|
3064
|
+
.value_or(GRPC_STATUS_UNKNOWN)));
|
3065
|
+
if (const Slice* message =
|
3066
|
+
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3067
|
+
absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
|
3068
|
+
}
|
3069
|
+
if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
|
3070
|
+
absl::StrAppend(&out, "\nStatus Context:");
|
3071
|
+
for (const std::string& annotation : *annotations) {
|
3072
|
+
absl::StrAppend(&out, "\n ", annotation);
|
3073
|
+
}
|
3074
|
+
}
|
3075
|
+
return out;
|
3076
|
+
}
|
3077
|
+
} // namespace
|
3078
|
+
|
3079
|
+
void ClientPromiseBasedCall::PublishStatus(
|
3080
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
3081
|
+
ServerMetadataHandle trailing_metadata) {
|
3082
|
+
const grpc_status_code status = trailing_metadata->get(GrpcStatusMetadata())
|
3083
|
+
.value_or(GRPC_STATUS_UNKNOWN);
|
3084
|
+
*op_args.status = status;
|
3085
|
+
absl::string_view message_string;
|
3086
|
+
if (Slice* message = trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3087
|
+
message_string = message->as_string_view();
|
3088
|
+
*op_args.status_details = message->Ref().TakeCSlice();
|
3089
|
+
} else {
|
3090
|
+
*op_args.status_details = grpc_empty_slice();
|
3091
|
+
}
|
3092
|
+
if (message_string.empty()) {
|
3093
|
+
RunFinalization(status, nullptr);
|
3094
|
+
} else {
|
3095
|
+
std::string error_string(message_string);
|
3096
|
+
RunFinalization(status, error_string.c_str());
|
3097
|
+
}
|
3098
|
+
if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
|
3099
|
+
*op_args.error_string =
|
3100
|
+
gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
|
3101
|
+
}
|
3102
|
+
PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata);
|
3103
|
+
// Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding
|
3104
|
+
// (so we don't call through twice)
|
3105
|
+
recv_status_on_client_ = absl::monostate();
|
3106
|
+
FinishOpOnCompletion(&recv_status_on_client_completion_,
|
3107
|
+
PendingOp::kReceiveStatusOnClient);
|
3108
|
+
}
|
3109
|
+
#endif
|
3110
|
+
|
3111
|
+
///////////////////////////////////////////////////////////////////////////////
|
3112
|
+
// ServerPromiseBasedCall
|
3113
|
+
|
3114
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3115
|
+
|
3116
|
+
class ServerPromiseBasedCall final : public PromiseBasedCall {
|
3117
|
+
public:
|
3118
|
+
ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
|
3119
|
+
|
3120
|
+
void CancelWithErrorLocked(grpc_error_handle) override
|
3121
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3122
|
+
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3123
|
+
bool is_notify_tag_closure) override;
|
3124
|
+
bool failed_before_recv_message() const override { abort(); }
|
3125
|
+
bool is_trailers_only() const override { abort(); }
|
3126
|
+
absl::string_view GetServerAuthority() const override { return ""; }
|
3127
|
+
|
3128
|
+
// Polling order for the server promise stack:
|
3129
|
+
//
|
3130
|
+
// │ ┌───────────────────────────────────────┐
|
3131
|
+
// │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management,
|
3132
|
+
// │ ├───────────────────────────────────────┤ signal call end to app
|
3133
|
+
// │ │ ConnectedChannel ├─┐
|
3134
|
+
// │ ├───────────────────────────────────────┤ └► Interactions with the
|
3135
|
+
// │ │ ... closest to transport filter │ transport - send/recv msgs
|
3136
|
+
// │ ├───────────────────────────────────────┤ and metadata, call phase
|
3137
|
+
// │ │ ... │ ordering
|
3138
|
+
// │ ├───────────────────────────────────────┤
|
3139
|
+
// │ │ ... closest to app filter │ ┌► Request matching, initial
|
3140
|
+
// │ ├───────────────────────────────────────┤ │ setup, publishing call to
|
3141
|
+
// │ │ Server::ChannelData::MakeCallPromise ├─┘ application
|
3142
|
+
// │ ├───────────────────────────────────────┤
|
3143
|
+
// │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions,
|
3144
|
+
// ▼ └───────────────────────────────────────┘ forwarding messages,
|
3145
|
+
// Polling & sending trailing metadata
|
3146
|
+
// instantiation
|
3147
|
+
// order
|
3148
|
+
|
3149
|
+
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
3150
|
+
Poll<ServerMetadataHandle> PollTopOfCall()
|
3151
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3152
|
+
|
3153
|
+
std::string DebugTag() const override {
|
3154
|
+
return absl::StrFormat("SERVER_CALL[%p]: ", this);
|
3155
|
+
}
|
3156
|
+
|
3157
|
+
ServerCallContext* server_call_context() override { return &call_context_; }
|
3158
|
+
|
3159
|
+
private:
|
3160
|
+
class RecvCloseOpCancelState {
|
3161
|
+
public:
|
3162
|
+
// Request that receiver be filled in per grpc_op_recv_close_on_server.
|
3163
|
+
// Returns true if the request can be fulfilled immediately.
|
3164
|
+
// Returns false if the request will be fulfilled later.
|
3165
|
+
bool ReceiveCloseOnServerOpStarted(int* receiver) {
|
3166
|
+
switch (state_) {
|
3167
|
+
case kUnset:
|
3168
|
+
state_ = reinterpret_cast<uintptr_t>(receiver);
|
3169
|
+
return false;
|
3170
|
+
case kFinishedWithFailure:
|
3171
|
+
*receiver = 1;
|
3172
|
+
return true;
|
3173
|
+
case kFinishedWithSuccess:
|
3174
|
+
*receiver = 0;
|
3175
|
+
return true;
|
3176
|
+
default:
|
3177
|
+
abort(); // unreachable
|
3178
|
+
}
|
3179
|
+
}
|
3180
|
+
|
3181
|
+
// Mark the call as having completed.
|
3182
|
+
// Returns true if this finishes a previous RequestReceiveCloseOnServer.
|
3183
|
+
bool CompleteCall(bool success) {
|
3184
|
+
switch (state_) {
|
3185
|
+
case kUnset:
|
3186
|
+
state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
|
3187
|
+
return false;
|
3188
|
+
case kFinishedWithFailure:
|
3189
|
+
case kFinishedWithSuccess:
|
3190
|
+
abort(); // unreachable
|
3191
|
+
default:
|
3192
|
+
*reinterpret_cast<int*>(state_) = success ? 0 : 1;
|
3193
|
+
state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
|
3194
|
+
return true;
|
3195
|
+
}
|
3196
|
+
}
|
3197
|
+
|
3198
|
+
std::string ToString() const {
|
3199
|
+
switch (state_) {
|
3200
|
+
case kUnset:
|
3201
|
+
return "Unset";
|
3202
|
+
case kFinishedWithFailure:
|
3203
|
+
return "FinishedWithFailure";
|
3204
|
+
case kFinishedWithSuccess:
|
3205
|
+
return "FinishedWithSuccess";
|
3206
|
+
default:
|
3207
|
+
return absl::StrFormat("WaitingForReceiver(%p)",
|
3208
|
+
reinterpret_cast<void*>(state_));
|
3209
|
+
}
|
3210
|
+
}
|
3211
|
+
|
3212
|
+
private:
|
3213
|
+
static constexpr uintptr_t kUnset = 0;
|
3214
|
+
static constexpr uintptr_t kFinishedWithFailure = 1;
|
3215
|
+
static constexpr uintptr_t kFinishedWithSuccess = 2;
|
3216
|
+
// Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess
|
3217
|
+
// OR an int* that wants to receive the final status.
|
3218
|
+
uintptr_t state_ = kUnset;
|
3219
|
+
};
|
3220
|
+
|
3221
|
+
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
3222
|
+
void CommitBatch(const grpc_op* ops, size_t nops,
|
3223
|
+
const Completion& completion)
|
3224
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3225
|
+
|
3226
|
+
friend class ServerCallContext;
|
3227
|
+
ServerCallContext call_context_;
|
3228
|
+
Server* const server_;
|
3229
|
+
ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
|
3230
|
+
PipeSender<MessageHandle>* server_to_client_messages_ ABSL_GUARDED_BY(mu()) =
|
3231
|
+
nullptr;
|
3232
|
+
PipeReceiver<MessageHandle>* client_to_server_messages_
|
3233
|
+
ABSL_GUARDED_BY(mu()) = nullptr;
|
3234
|
+
using SendInitialMetadataState =
|
3235
|
+
absl::variant<absl::monostate, PipeSender<ServerMetadataHandle>*,
|
3236
|
+
typename PipeSender<ServerMetadataHandle>::PushType>;
|
3237
|
+
SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) =
|
3238
|
+
absl::monostate{};
|
3239
|
+
ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu());
|
3240
|
+
grpc_compression_algorithm incoming_compression_algorithm_
|
3241
|
+
ABSL_GUARDED_BY(mu());
|
3242
|
+
RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu());
|
3243
|
+
Completion recv_close_completion_ ABSL_GUARDED_BY(mu());
|
3244
|
+
bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false;
|
3245
|
+
Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu());
|
3246
|
+
ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu());
|
3247
|
+
};
|
3248
|
+
|
3249
|
+
ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
3250
|
+
grpc_call_create_args* args)
|
3251
|
+
: PromiseBasedCall(arena, 0, *args),
|
3252
|
+
call_context_(this, args->server_transport_data),
|
3253
|
+
server_(args->server) {
|
3254
|
+
global_stats().IncrementServerCallsCreated();
|
3255
|
+
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3256
|
+
if (channelz_node != nullptr) {
|
3257
|
+
channelz_node->RecordCallStarted();
|
3258
|
+
}
|
3259
|
+
MutexLock lock(mu());
|
3260
|
+
ScopedContext activity_context(this);
|
3261
|
+
promise_ = channel()->channel_stack()->MakeServerCallPromise(
|
3262
|
+
CallArgs{nullptr, nullptr, nullptr, nullptr});
|
3263
|
+
}
|
3264
|
+
|
3265
|
+
Poll<ServerMetadataHandle> ServerPromiseBasedCall::PollTopOfCall() {
|
3266
|
+
if (grpc_call_trace.enabled()) {
|
3267
|
+
gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(),
|
3268
|
+
cancel_send_and_receive_ ? "force-" : "",
|
3269
|
+
send_trailing_metadata_ != nullptr
|
3270
|
+
? absl::StrCat("send-metadata:",
|
3271
|
+
send_trailing_metadata_->DebugString(), " ")
|
3272
|
+
.c_str()
|
3273
|
+
: " ",
|
3274
|
+
PollStateDebugString().c_str());
|
3275
|
+
}
|
3276
|
+
|
3277
|
+
if (cancel_send_and_receive_) {
|
3278
|
+
CancelSendMessage();
|
3279
|
+
CancelRecvMessage();
|
3280
|
+
}
|
3281
|
+
|
3282
|
+
PollSendMessage();
|
3283
|
+
PollRecvMessage(incoming_compression_algorithm_);
|
3284
|
+
|
3285
|
+
if (!is_sending() && send_trailing_metadata_ != nullptr) {
|
3286
|
+
server_to_client_messages_->Close();
|
3287
|
+
return std::move(send_trailing_metadata_);
|
3288
|
+
}
|
3289
|
+
|
3290
|
+
return Pending{};
|
3291
|
+
}
|
3292
|
+
|
3293
|
+
void ServerPromiseBasedCall::UpdateOnce() {
|
3294
|
+
if (grpc_call_trace.enabled()) {
|
3295
|
+
gpr_log(
|
3296
|
+
GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s",
|
3297
|
+
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3298
|
+
recv_close_completion_.has_value()
|
3299
|
+
? absl::StrCat(":", CompletionString(recv_close_completion_))
|
3300
|
+
.c_str()
|
3301
|
+
: "",
|
3302
|
+
send_status_from_server_completion_.has_value()
|
3303
|
+
? absl::StrCat(
|
3304
|
+
"send_status:",
|
3305
|
+
CompletionString(send_status_from_server_completion_), " ")
|
3306
|
+
.c_str()
|
3307
|
+
: "",
|
3308
|
+
PollStateDebugString().c_str(),
|
3309
|
+
promise_.has_value() ? "true" : "false");
|
3310
|
+
}
|
3311
|
+
if (auto* p =
|
3312
|
+
absl::get_if<typename PipeSender<ServerMetadataHandle>::PushType>(
|
3313
|
+
&send_initial_metadata_state_)) {
|
3314
|
+
if ((*p)().ready()) {
|
3315
|
+
send_initial_metadata_state_ = absl::monostate{};
|
3316
|
+
}
|
3317
|
+
}
|
3318
|
+
if (promise_.has_value()) {
|
3319
|
+
auto r = promise_();
|
3320
|
+
if (grpc_call_trace.enabled()) {
|
3321
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
|
3322
|
+
DebugTag().c_str(),
|
3323
|
+
PollToString(r, [](const ServerMetadataHandle& h) {
|
3324
|
+
return h->DebugString();
|
3325
|
+
}).c_str());
|
3326
|
+
}
|
3327
|
+
if (auto* result = r.value_if_ready()) {
|
3328
|
+
if (grpc_call_trace.enabled()) {
|
3329
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s",
|
3330
|
+
DebugTag().c_str(),
|
3331
|
+
recv_close_op_cancel_state_.ToString().c_str(),
|
3332
|
+
(*result)->DebugString().c_str());
|
3333
|
+
}
|
3334
|
+
if (recv_close_op_cancel_state_.CompleteCall(
|
3335
|
+
(*result)->get(GrpcStatusFromWire()).value_or(false))) {
|
3336
|
+
FinishOpOnCompletion(&recv_close_completion_,
|
3337
|
+
PendingOp::kReceiveCloseOnServer);
|
3338
|
+
}
|
3339
|
+
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3340
|
+
if (channelz_node != nullptr) {
|
3341
|
+
if ((*result)
|
3342
|
+
->get(GrpcStatusMetadata())
|
3343
|
+
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3344
|
+
channelz_node->RecordCallSucceeded();
|
3345
|
+
} else {
|
3346
|
+
channelz_node->RecordCallFailed();
|
3347
|
+
}
|
3348
|
+
}
|
3349
|
+
if (send_status_from_server_completion_.has_value()) {
|
3350
|
+
FinishOpOnCompletion(&send_status_from_server_completion_,
|
3351
|
+
PendingOp::kSendStatusFromServer);
|
3352
|
+
}
|
3353
|
+
CancelSendMessage();
|
3354
|
+
CancelRecvMessage();
|
3355
|
+
set_completed();
|
3356
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3357
|
+
}
|
3358
|
+
}
|
3359
|
+
}
|
3360
|
+
|
3361
|
+
grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
3362
|
+
size_t nops) const {
|
3363
|
+
BitSet<8> got_ops;
|
3364
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3365
|
+
const grpc_op& op = ops[op_idx];
|
3366
|
+
switch (op.op) {
|
3367
|
+
case GRPC_OP_SEND_INITIAL_METADATA:
|
3368
|
+
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
3369
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3370
|
+
}
|
3371
|
+
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
3372
|
+
op.data.send_initial_metadata.metadata)) {
|
3373
|
+
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3374
|
+
}
|
3375
|
+
break;
|
3376
|
+
case GRPC_OP_SEND_MESSAGE:
|
3377
|
+
if (!AreWriteFlagsValid(op.flags)) {
|
3378
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3379
|
+
}
|
3380
|
+
break;
|
3381
|
+
case GRPC_OP_RECV_MESSAGE:
|
3382
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3383
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3384
|
+
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3385
|
+
break;
|
3386
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
3387
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3388
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3389
|
+
return GRPC_CALL_ERROR_NOT_ON_SERVER;
|
3390
|
+
}
|
3391
|
+
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
3392
|
+
got_ops.set(op.op);
|
3393
|
+
}
|
3394
|
+
return GRPC_CALL_OK;
|
3395
|
+
}
|
3396
|
+
|
3397
|
+
void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
3398
|
+
const Completion& completion) {
|
3399
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3400
|
+
const grpc_op& op = ops[op_idx];
|
3401
|
+
switch (op.op) {
|
3402
|
+
case GRPC_OP_SEND_INITIAL_METADATA: {
|
3403
|
+
// compression not implemented
|
3404
|
+
GPR_ASSERT(
|
3405
|
+
!op.data.send_initial_metadata.maybe_compression_level.is_set);
|
3406
|
+
if (!completed()) {
|
3407
|
+
auto metadata = arena()->MakePooled<ServerMetadata>(arena());
|
3408
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
3409
|
+
op.data.send_initial_metadata.count, metadata.get());
|
3410
|
+
if (grpc_call_trace.enabled()) {
|
3411
|
+
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3412
|
+
DebugTag().c_str());
|
3413
|
+
}
|
3414
|
+
auto* pipe = absl::get<PipeSender<ServerMetadataHandle>*>(
|
3415
|
+
send_initial_metadata_state_);
|
3416
|
+
send_initial_metadata_state_ = pipe->Push(std::move(metadata));
|
3417
|
+
}
|
3418
|
+
} break;
|
3419
|
+
case GRPC_OP_SEND_MESSAGE:
|
3420
|
+
StartSendMessage(op, completion, server_to_client_messages_);
|
3421
|
+
break;
|
3422
|
+
case GRPC_OP_RECV_MESSAGE:
|
3423
|
+
StartRecvMessage(op, completion, client_to_server_messages_);
|
3424
|
+
break;
|
3425
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3426
|
+
send_trailing_metadata_ = arena()->MakePooled<ServerMetadata>(arena());
|
3427
|
+
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3428
|
+
op.data.send_status_from_server.trailing_metadata_count,
|
3429
|
+
send_trailing_metadata_.get());
|
3430
|
+
send_trailing_metadata_->Set(GrpcStatusMetadata(),
|
3431
|
+
op.data.send_status_from_server.status);
|
3432
|
+
if (auto* details = op.data.send_status_from_server.status_details) {
|
3433
|
+
send_trailing_metadata_->Set(GrpcMessageMetadata(),
|
3434
|
+
Slice(CSliceRef(*details)));
|
3435
|
+
}
|
3436
|
+
send_status_from_server_completion_ =
|
3437
|
+
AddOpToCompletion(completion, PendingOp::kSendStatusFromServer);
|
3438
|
+
break;
|
3439
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3440
|
+
if (grpc_call_trace.enabled()) {
|
3441
|
+
gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
|
3442
|
+
DebugTag().c_str(),
|
3443
|
+
recv_close_op_cancel_state_.ToString().c_str());
|
3444
|
+
}
|
3445
|
+
if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
|
3446
|
+
op.data.recv_close_on_server.cancelled)) {
|
3447
|
+
recv_close_completion_ =
|
3448
|
+
AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
|
3449
|
+
}
|
3450
|
+
break;
|
3451
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3452
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3453
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
3454
|
+
abort(); // unreachable
|
3455
|
+
}
|
3456
|
+
}
|
3457
|
+
}
|
3458
|
+
|
3459
|
+
grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
3460
|
+
size_t nops,
|
3461
|
+
void* notify_tag,
|
3462
|
+
bool is_notify_tag_closure) {
|
3463
|
+
MutexLock lock(mu());
|
3464
|
+
ScopedContext activity_context(this);
|
3465
|
+
if (nops == 0) {
|
3466
|
+
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3467
|
+
return GRPC_CALL_OK;
|
3468
|
+
}
|
3469
|
+
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
3470
|
+
if (validation_result != GRPC_CALL_OK) {
|
3471
|
+
return validation_result;
|
3472
|
+
}
|
3473
|
+
Completion completion =
|
3474
|
+
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
3475
|
+
CommitBatch(ops, nops, completion);
|
3476
|
+
Update();
|
3477
|
+
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
3478
|
+
return GRPC_CALL_OK;
|
3479
|
+
}
|
3480
|
+
|
3481
|
+
void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) {
|
3482
|
+
if (!promise_.has_value()) return;
|
3483
|
+
cancel_send_and_receive_ = true;
|
3484
|
+
send_trailing_metadata_ = ServerMetadataFromStatus(error, arena());
|
3485
|
+
ForceWakeup();
|
3486
|
+
}
|
3487
|
+
|
3488
|
+
#endif
|
3489
|
+
|
3490
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3491
|
+
ArenaPromise<ServerMetadataHandle>
|
3492
|
+
ServerCallContext::MakeTopOfServerCallPromise(
|
3493
|
+
CallArgs call_args, grpc_completion_queue* cq,
|
3494
|
+
grpc_metadata_array* publish_initial_metadata,
|
3495
|
+
absl::FunctionRef<void(grpc_call* call)> publish) {
|
3496
|
+
call_->mu()->AssertHeld();
|
3497
|
+
call_->SetCompletionQueueLocked(cq);
|
3498
|
+
call_->server_to_client_messages_ = call_args.server_to_client_messages;
|
3499
|
+
call_->client_to_server_messages_ = call_args.client_to_server_messages;
|
3500
|
+
call_->send_initial_metadata_state_ = call_args.server_initial_metadata;
|
3501
|
+
call_->incoming_compression_algorithm_ =
|
3502
|
+
call_args.client_initial_metadata->get(GrpcEncodingMetadata())
|
3503
|
+
.value_or(GRPC_COMPRESS_NONE);
|
3504
|
+
call_->client_initial_metadata_ =
|
3505
|
+
std::move(call_args.client_initial_metadata);
|
3506
|
+
PublishMetadataArray(call_->client_initial_metadata_.get(),
|
3507
|
+
publish_initial_metadata);
|
3508
|
+
call_->ExternalRef();
|
3509
|
+
publish(call_->c_ptr());
|
3510
|
+
return [this]() {
|
3511
|
+
call_->mu()->AssertHeld();
|
3512
|
+
return call_->PollTopOfCall();
|
3513
|
+
};
|
3514
|
+
}
|
3515
|
+
#else
|
3516
|
+
ArenaPromise<ServerMetadataHandle>
|
3517
|
+
ServerCallContext::MakeTopOfServerCallPromise(
|
3518
|
+
CallArgs, grpc_completion_queue*, grpc_metadata_array*,
|
3519
|
+
absl::FunctionRef<void(grpc_call*)>) {
|
3520
|
+
(void)call_;
|
3521
|
+
Crash("Promise-based server call is not enabled");
|
3522
|
+
}
|
3523
|
+
#endif
|
3524
|
+
|
1798
3525
|
} // namespace grpc_core
|
1799
3526
|
|
3527
|
+
///////////////////////////////////////////////////////////////////////////////
|
3528
|
+
// C-based API
|
3529
|
+
|
1800
3530
|
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
1801
3531
|
grpc_core::ExecCtx exec_ctx;
|
1802
3532
|
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
@@ -1808,6 +3538,20 @@ size_t grpc_call_get_initial_size_estimate() {
|
|
1808
3538
|
|
1809
3539
|
grpc_error_handle grpc_call_create(grpc_call_create_args* args,
|
1810
3540
|
grpc_call** out_call) {
|
3541
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
3542
|
+
if (grpc_core::IsPromiseBasedClientCallEnabled() &&
|
3543
|
+
args->server_transport_data == nullptr && args->channel->is_promising()) {
|
3544
|
+
return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
|
3545
|
+
args, out_call);
|
3546
|
+
}
|
3547
|
+
#endif
|
3548
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3549
|
+
if (grpc_core::IsPromiseBasedServerCallEnabled() &&
|
3550
|
+
args->server_transport_data != nullptr && args->channel->is_promising()) {
|
3551
|
+
return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
|
3552
|
+
args, out_call);
|
3553
|
+
}
|
3554
|
+
#endif
|
1811
3555
|
return grpc_core::FilterStackCall::Create(args, out_call);
|
1812
3556
|
}
|
1813
3557
|
|
@@ -1819,6 +3563,7 @@ void grpc_call_set_completion_queue(grpc_call* call,
|
|
1819
3563
|
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
1820
3564
|
|
1821
3565
|
void grpc_call_unref(grpc_call* c) {
|
3566
|
+
grpc_core::ExecCtx exec_ctx;
|
1822
3567
|
grpc_core::Call::FromC(c)->ExternalUnref();
|
1823
3568
|
}
|
1824
3569
|
|
@@ -1835,7 +3580,7 @@ grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
|
|
1835
3580
|
GPR_ASSERT(reserved == nullptr);
|
1836
3581
|
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
1837
3582
|
grpc_core::ExecCtx exec_ctx;
|
1838
|
-
grpc_core::Call::FromC(call)->CancelWithError(
|
3583
|
+
grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
|
1839
3584
|
return GRPC_CALL_OK;
|
1840
3585
|
}
|
1841
3586
|
|
@@ -1855,7 +3600,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call* c,
|
|
1855
3600
|
}
|
1856
3601
|
|
1857
3602
|
void grpc_call_cancel_internal(grpc_call* call) {
|
1858
|
-
grpc_core::Call::FromC(call)->CancelWithError(
|
3603
|
+
grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
|
1859
3604
|
}
|
1860
3605
|
|
1861
3606
|
grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
|
@@ -1928,6 +3673,10 @@ int grpc_call_failed_before_recv_message(const grpc_call* c) {
|
|
1928
3673
|
return grpc_core::Call::FromC(c)->failed_before_recv_message();
|
1929
3674
|
}
|
1930
3675
|
|
3676
|
+
absl::string_view grpc_call_server_authority(const grpc_call* call) {
|
3677
|
+
return grpc_core::Call::FromC(call)->GetServerAuthority();
|
3678
|
+
}
|
3679
|
+
|
1931
3680
|
const char* grpc_call_error_to_string(grpc_call_error error) {
|
1932
3681
|
switch (error) {
|
1933
3682
|
case GRPC_CALL_ERROR:
|