grpc 1.45.0 → 1.53.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of grpc might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Makefile +437 -254
- data/include/grpc/byte_buffer.h +76 -1
- data/include/grpc/byte_buffer_reader.h +19 -1
- data/include/grpc/compression.h +2 -2
- data/include/grpc/event_engine/endpoint_config.h +11 -5
- data/include/grpc/event_engine/event_engine.h +129 -39
- data/include/grpc/event_engine/internal/memory_allocator_impl.h +1 -1
- data/include/grpc/event_engine/internal/slice_cast.h +67 -0
- data/include/grpc/event_engine/memory_allocator.h +1 -16
- data/include/grpc/event_engine/port.h +1 -1
- data/include/grpc/event_engine/slice.h +306 -0
- data/include/grpc/event_engine/slice_buffer.h +159 -0
- data/include/grpc/fork.h +25 -1
- data/include/grpc/grpc.h +4 -14
- data/include/grpc/grpc_posix.h +1 -1
- data/include/grpc/grpc_security.h +11 -0
- data/include/grpc/impl/codegen/atm.h +3 -71
- data/include/grpc/impl/codegen/atm_gcc_atomic.h +3 -67
- data/include/grpc/impl/codegen/atm_gcc_sync.h +3 -61
- data/include/grpc/impl/codegen/atm_windows.h +3 -108
- data/include/grpc/impl/codegen/byte_buffer.h +4 -78
- data/include/grpc/impl/codegen/byte_buffer_reader.h +4 -19
- data/include/grpc/impl/codegen/compression_types.h +5 -83
- data/include/grpc/impl/codegen/connectivity_state.h +5 -21
- data/include/grpc/impl/codegen/fork.h +4 -25
- data/include/grpc/impl/codegen/gpr_types.h +4 -35
- data/include/grpc/impl/codegen/grpc_types.h +5 -797
- data/include/grpc/impl/codegen/log.h +3 -86
- data/include/grpc/impl/codegen/port_platform.h +3 -699
- data/include/grpc/impl/codegen/propagation_bits.h +3 -28
- data/include/grpc/impl/codegen/slice.h +4 -107
- data/include/grpc/impl/codegen/status.h +4 -131
- data/include/grpc/impl/codegen/sync.h +3 -42
- data/include/grpc/impl/codegen/sync_abseil.h +3 -12
- data/include/grpc/impl/codegen/sync_custom.h +3 -14
- data/include/grpc/impl/codegen/sync_generic.h +3 -25
- data/include/grpc/impl/codegen/sync_posix.h +3 -28
- data/include/grpc/impl/codegen/sync_windows.h +3 -16
- data/include/grpc/impl/compression_types.h +109 -0
- data/include/grpc/impl/connectivity_state.h +47 -0
- data/include/grpc/impl/grpc_types.h +827 -0
- data/include/grpc/impl/propagation_bits.h +54 -0
- data/include/grpc/impl/slice_type.h +112 -0
- data/include/grpc/load_reporting.h +1 -1
- data/include/grpc/module.modulemap +5 -1
- data/include/grpc/slice.h +1 -1
- data/include/grpc/status.h +131 -1
- data/include/grpc/support/atm.h +70 -1
- data/include/grpc/support/atm_gcc_atomic.h +59 -1
- data/include/grpc/support/atm_gcc_sync.h +58 -1
- data/include/grpc/support/atm_windows.h +105 -1
- data/include/grpc/support/log.h +87 -1
- data/include/grpc/support/log_windows.h +1 -1
- data/include/grpc/support/port_platform.h +767 -1
- data/include/grpc/support/string_util.h +1 -1
- data/include/grpc/support/sync.h +35 -2
- data/include/grpc/support/sync_abseil.h +11 -1
- data/include/grpc/support/sync_custom.h +13 -1
- data/include/grpc/support/sync_generic.h +24 -1
- data/include/grpc/support/sync_posix.h +27 -1
- data/include/grpc/support/sync_windows.h +15 -1
- data/include/grpc/support/time.h +31 -6
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +148 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +52 -0
- data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +29 -0
- data/src/core/ext/filters/census/grpc_context.cc +19 -17
- data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +315 -0
- data/src/core/ext/filters/channel_idle/channel_idle_filter.h +142 -0
- data/src/core/ext/filters/{client_idle → channel_idle}/idle_filter_state.cc +1 -1
- data/src/core/ext/filters/{client_idle → channel_idle}/idle_filter_state.h +6 -4
- data/src/core/ext/filters/client_channel/backend_metric.cc +19 -12
- data/src/core/ext/filters/client_channel/backend_metric.h +21 -11
- data/src/core/ext/filters/client_channel/backup_poller.cc +34 -34
- data/src/core/ext/filters/client_channel/backup_poller.h +26 -28
- data/src/core/ext/filters/client_channel/channel_connectivity.cc +89 -44
- data/src/core/ext/filters/client_channel/client_channel.cc +557 -540
- data/src/core/ext/filters/client_channel/client_channel.h +70 -26
- data/src/core/ext/filters/client_channel/client_channel_channelz.cc +19 -23
- data/src/core/ext/filters/client_channel/client_channel_channelz.h +33 -23
- data/src/core/ext/filters/client_channel/client_channel_factory.cc +19 -43
- data/src/core/ext/filters/client_channel/client_channel_factory.h +11 -11
- data/src/core/ext/filters/client_channel/client_channel_plugin.cc +22 -47
- data/src/core/ext/filters/client_channel/client_channel_service_config.cc +153 -0
- data/src/core/ext/filters/client_channel/{resolver_result_parsing.h → client_channel_service_config.h} +33 -27
- data/src/core/ext/filters/client_channel/config_selector.cc +1 -0
- data/src/core/ext/filters/client_channel/config_selector.h +29 -23
- data/src/core/ext/filters/client_channel/connector.h +19 -10
- data/src/core/ext/filters/client_channel/dynamic_filters.cc +40 -54
- data/src/core/ext/filters/client_channel/dynamic_filters.h +20 -11
- data/src/core/ext/filters/client_channel/global_subchannel_pool.cc +2 -0
- data/src/core/ext/filters/client_channel/global_subchannel_pool.h +24 -21
- data/src/core/ext/filters/client_channel/health/health_check_client.cc +137 -582
- data/src/core/ext/filters/client_channel/health/health_check_client.h +29 -163
- data/src/core/ext/filters/client_channel/http_proxy.cc +130 -164
- data/src/core/ext/filters/client_channel/http_proxy.h +46 -22
- data/src/core/ext/filters/client_channel/lb_call_state_internal.h +39 -0
- data/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc +8 -5
- data/src/core/ext/filters/client_channel/lb_policy/address_filtering.h +4 -3
- data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +49 -0
- data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc +37 -16
- data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h +18 -13
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +71 -131
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h +41 -24
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +410 -375
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h +26 -27
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc +15 -3
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h +9 -7
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc +18 -20
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +27 -22
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc +28 -19
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +26 -25
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc +338 -0
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h +57 -0
- data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric_internal.h +117 -0
- data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc +1129 -0
- data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h +94 -0
- data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +174 -180
- data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc +478 -479
- data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +491 -409
- data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h +19 -9
- data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +618 -642
- data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +261 -234
- data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +165 -143
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc +128 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.h +71 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc +972 -0
- data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +280 -248
- data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +251 -187
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_attributes.cc +42 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/{xds.h → xds_attributes.h} +19 -15
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h +3 -3
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +178 -256
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +212 -216
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +543 -584
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +819 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h +67 -0
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc +369 -0
- data/src/core/ext/filters/client_channel/local_subchannel_pool.cc +4 -0
- data/src/core/ext/filters/client_channel/local_subchannel_pool.h +22 -21
- data/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc +34 -19
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +579 -376
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +50 -44
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +31 -30
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +89 -84
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +300 -209
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +82 -61
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +18 -18
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +20 -19
- data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +2 -0
- data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h +22 -22
- data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +86 -224
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +21 -34
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +19 -5
- data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +135 -63
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +269 -0
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +121 -0
- data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +21 -27
- data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +413 -303
- data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h +6 -4
- data/src/core/ext/filters/client_channel/retry_filter.cc +184 -166
- data/src/core/ext/filters/client_channel/retry_filter.h +4 -3
- data/src/core/ext/filters/client_channel/retry_service_config.cc +197 -229
- data/src/core/ext/filters/client_channel/retry_service_config.h +29 -26
- data/src/core/ext/filters/client_channel/retry_throttle.cc +28 -37
- data/src/core/ext/filters/client_channel/retry_throttle.h +36 -29
- data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +34 -17
- data/src/core/ext/filters/client_channel/subchannel.cc +336 -388
- data/src/core/ext/filters/client_channel/subchannel.h +112 -82
- data/src/core/ext/filters/client_channel/subchannel_interface_internal.h +38 -0
- data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc +17 -77
- data/src/core/ext/filters/client_channel/subchannel_pool_interface.h +41 -47
- data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +470 -0
- data/src/core/ext/filters/client_channel/subchannel_stream_client.h +222 -0
- data/src/core/ext/filters/deadline/deadline_filter.cc +94 -76
- data/src/core/ext/filters/deadline/deadline_filter.h +14 -12
- data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +154 -372
- data/src/core/ext/filters/fault_injection/fault_injection_filter.h +41 -9
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc +118 -0
- data/src/core/ext/filters/fault_injection/{service_config_parser.h → fault_injection_service_config_parser.h} +34 -13
- data/src/core/ext/filters/http/client/http_client_filter.cc +119 -477
- data/src/core/ext/filters/http/client/http_client_filter.h +55 -25
- data/src/core/ext/filters/http/client_authority_filter.cc +48 -58
- data/src/core/ext/filters/http/client_authority_filter.h +32 -28
- data/src/core/ext/filters/http/http_filters_plugin.cc +46 -52
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +307 -0
- data/src/core/ext/filters/http/message_compress/compression_filter.h +139 -0
- data/src/core/ext/filters/http/server/http_server_filter.cc +111 -385
- data/src/core/ext/filters/http/server/http_server_filter.h +58 -24
- data/src/core/ext/filters/message_size/message_size_filter.cc +117 -137
- data/src/core/ext/filters/message_size/message_size_filter.h +34 -17
- data/src/core/ext/filters/rbac/rbac_filter.cc +29 -16
- data/src/core/ext/filters/rbac/rbac_filter.h +11 -3
- data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +736 -523
- data/src/core/ext/filters/rbac/rbac_service_config_parser.h +18 -7
- data/src/core/ext/filters/server_config_selector/server_config_selector.h +20 -8
- data/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +90 -202
- data/src/core/ext/filters/server_config_selector/server_config_selector_filter.h +4 -3
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +219 -0
- data/src/core/ext/filters/stateful_session/stateful_session_filter.h +66 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc +82 -0
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +93 -0
- data/src/core/ext/transport/chttp2/alpn/alpn.cc +18 -18
- data/src/core/ext/transport/chttp2/alpn/alpn.h +24 -24
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +186 -241
- data/src/core/ext/transport/chttp2/client/chttp2_connector.h +38 -30
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +209 -215
- data/src/core/ext/transport/chttp2/server/chttp2_server.h +26 -26
- data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +24 -24
- data/src/core/ext/transport/chttp2/transport/bin_decoder.h +35 -34
- data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +31 -30
- data/src/core/ext/transport/chttp2/transport/bin_encoder.h +29 -29
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +893 -1079
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +28 -23
- data/src/core/ext/transport/chttp2/transport/context_list.cc +23 -19
- data/src/core/ext/transport/chttp2/transport/context_list.h +32 -29
- data/src/core/ext/transport/chttp2/transport/decode_huff.cc +251 -0
- data/src/core/ext/transport/chttp2/transport/decode_huff.h +971 -0
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +350 -300
- data/src/core/ext/transport/chttp2/transport/flow_control.h +248 -325
- data/src/core/ext/transport/chttp2/transport/frame.h +22 -26
- data/src/core/ext/transport/chttp2/transport/frame_data.cc +85 -234
- data/src/core/ext/transport/chttp2/transport/frame_data.h +38 -61
- data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +41 -36
- data/src/core/ext/transport/chttp2/transport/frame_goaway.h +24 -22
- data/src/core/ext/transport/chttp2/transport/frame_ping.cc +31 -25
- data/src/core/ext/transport/chttp2/transport/frame_ping.h +25 -22
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +35 -26
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +24 -21
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +46 -77
- data/src/core/ext/transport/chttp2/transport/frame_settings.h +27 -23
- data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +34 -33
- data/src/core/ext/transport/chttp2/transport/frame_window_update.h +24 -21
- data/src/core/ext/transport/chttp2/transport/hpack_constants.h +10 -4
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +169 -220
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +59 -61
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +4 -1
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +14 -5
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +114 -542
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +31 -24
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +75 -67
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +70 -48
- data/src/core/ext/transport/chttp2/transport/http2_settings.cc +21 -19
- data/src/core/ext/transport/chttp2/transport/http2_settings.h +39 -38
- data/src/core/ext/transport/chttp2/transport/http_trace.cc +19 -0
- data/src/core/ext/transport/chttp2/transport/http_trace.h +24 -0
- data/src/core/ext/transport/chttp2/transport/huffsyms.cc +20 -20
- data/src/core/ext/transport/chttp2/transport/huffsyms.h +21 -21
- data/src/core/ext/transport/chttp2/transport/internal.h +250 -298
- data/src/core/ext/transport/chttp2/transport/parsing.cc +365 -132
- data/src/core/ext/transport/chttp2/transport/stream_lists.cc +30 -30
- data/src/core/ext/transport/chttp2/transport/stream_map.cc +24 -24
- data/src/core/ext/transport/chttp2/transport/stream_map.h +34 -33
- data/src/core/ext/transport/chttp2/transport/varint.cc +19 -20
- data/src/core/ext/transport/chttp2/transport/varint.h +39 -34
- data/src/core/ext/transport/chttp2/transport/writing.cc +144 -106
- data/src/core/ext/transport/inproc/inproc_plugin.cc +17 -18
- data/src/core/ext/transport/inproc/inproc_transport.cc +227 -274
- data/src/core/ext/transport/inproc/inproc_transport.h +23 -21
- data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c +27 -24
- data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.h +173 -73
- data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c +39 -36
- data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.h +210 -85
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c +29 -344
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.h +189 -1489
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.c +388 -0
- data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.h +1953 -0
- data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.h +38 -17
- data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c +9 -7
- data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.h +56 -13
- data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c +7 -7
- data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.h +24 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.h +19 -6
- data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.h +15 -2
- data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.c +49 -46
- data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.h +229 -82
- data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.h +16 -4
- data/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.c +12 -12
- data/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.h +40 -8
- data/src/core/ext/upb-generated/envoy/annotations/resource.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/annotations/resource.upb.h +26 -4
- data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c +55 -55
- data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.h +379 -140
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +140 -126
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +824 -301
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c +24 -18
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.h +114 -29
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c +195 -191
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.h +1042 -412
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.h +16 -4
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c +44 -24
- data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.h +171 -42
- data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c +72 -63
- data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.h +517 -202
- data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c +61 -36
- data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.h +333 -66
- data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c +5 -4
- data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.h +20 -6
- data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c +93 -91
- data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.h +615 -213
- data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c +76 -33
- data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.h +417 -79
- data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.h +20 -8
- data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c +5 -25
- data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.h +16 -93
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.h +38 -17
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c +70 -68
- data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.h +447 -157
- data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c +98 -66
- data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.h +545 -191
- data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.h +33 -13
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +108 -87
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +502 -149
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +9 -2
- data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c +6 -6
- data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.h +42 -17
- data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c +24 -9
- data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.h +110 -18
- data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.h +64 -26
- data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.h +20 -6
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c +18 -18
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.h +118 -49
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c +39 -32
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.h +242 -87
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c +32 -32
- data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.h +226 -105
- data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.h +13 -4
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +86 -53
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +475 -141
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c +56 -50
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.h +379 -160
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +16 -12
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +77 -16
- data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c +9 -6
- data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.h +50 -8
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.h +26 -6
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c +33 -32
- data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.h +253 -102
- data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c +34 -34
- data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.h +229 -92
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +74 -55
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +494 -160
- data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c +38 -30
- data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.h +232 -124
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +458 -351
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +2879 -947
- data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c +11 -11
- data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.h +67 -22
- data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c +79 -54
- data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.h +454 -133
- data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c +5 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.h +21 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.h +16 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.h +38 -14
- data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.h +33 -14
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c +17 -16
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.h +89 -38
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c +47 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.h +107 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.h +13 -4
- data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.h +58 -21
- data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.c +1 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.h +1 -0
- data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c +9 -9
- data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.h +50 -16
- data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c +9 -8
- data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.h +43 -10
- data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +16 -8
- data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c +15 -15
- data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.h +101 -38
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c +25 -25
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.h +167 -71
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c +16 -8
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +88 -16
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c +14 -11
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.h +71 -17
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c +62 -0
- data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h +179 -0
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +184 -144
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +1091 -361
- data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c +46 -0
- data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h +98 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c +54 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +188 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.c +111 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.h +444 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c +56 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +192 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c +46 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h +98 -0
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c +56 -48
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.h +327 -130
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c +13 -13
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +88 -31
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c +83 -56
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +472 -136
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +35 -13
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c +1 -1
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.h +6 -2
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c +171 -54
- data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.h +917 -134
- data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.h +59 -25
- data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c +33 -33
- data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.h +205 -76
- data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.h +23 -8
- data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.h +62 -25
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.c +47 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.h +113 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +36 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c +7 -7
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.h +55 -22
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.h +26 -11
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c +3 -3
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.h +30 -11
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.h +20 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c +8 -8
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.h +58 -22
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.c +43 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.h +114 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c +9 -9
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.h +82 -29
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.h +49 -22
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c +12 -12
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.h +99 -36
- data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c +16 -16
- data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.h +118 -44
- data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c +20 -20
- data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.h +120 -40
- data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c +7 -7
- data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.h +46 -16
- data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c +2 -2
- data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.h +9 -2
- data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.h +25 -8
- data/src/core/ext/upb-generated/envoy/type/v3/range.upb.c +9 -9
- data/src/core/ext/upb-generated/envoy/type/v3/range.upb.h +36 -6
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.c +63 -0
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.h +202 -0
- data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_unit.upb.h +3 -1
- data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c +4 -4
- data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.h +15 -2
- data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.c +5 -5
- data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.h +23 -6
- data/src/core/ext/upb-generated/google/api/annotations.upb.c +3 -3
- data/src/core/ext/upb-generated/google/api/annotations.upb.h +17 -2
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c +70 -66
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.h +477 -185
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c +80 -74
- data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h +512 -182
- data/src/core/ext/upb-generated/google/api/http.upb.c +17 -17
- data/src/core/ext/upb-generated/google/api/http.upb.h +134 -49
- data/src/core/ext/upb-generated/google/api/httpbody.upb.c +4 -4
- data/src/core/ext/upb-generated/google/api/httpbody.upb.h +25 -9
- data/src/core/ext/upb-generated/google/protobuf/any.upb.c +3 -3
- data/src/core/ext/upb-generated/google/protobuf/any.upb.h +12 -2
- data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c +183 -181
- data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.h +1332 -488
- data/src/core/ext/upb-generated/google/protobuf/duration.upb.c +3 -3
- data/src/core/ext/upb-generated/google/protobuf/duration.upb.h +16 -6
- data/src/core/ext/upb-generated/google/protobuf/empty.upb.c +1 -1
- data/src/core/ext/upb-generated/google/protobuf/empty.upb.h +6 -2
- data/src/core/ext/upb-generated/google/protobuf/struct.upb.c +15 -15
- data/src/core/ext/upb-generated/google/protobuf/struct.upb.h +126 -46
- data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c +3 -3
- data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.h +16 -6
- data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c +18 -18
- data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.h +81 -18
- data/src/core/ext/upb-generated/google/rpc/status.upb.c +4 -4
- data/src/core/ext/upb-generated/google/rpc/status.upb.h +25 -9
- data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c +15 -15
- data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.h +89 -30
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c +12 -12
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.h +55 -14
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c +69 -65
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.h +424 -169
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c +8 -7
- data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.h +32 -8
- data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c +4 -4
- data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.h +18 -4
- data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c +31 -31
- data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.h +204 -76
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c +11 -11
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.h +63 -22
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c +175 -0
- data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.h +792 -0
- data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c +22 -22
- data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.h +115 -16
- data/src/core/ext/upb-generated/udpa/annotations/security.upb.c +6 -6
- data/src/core/ext/upb-generated/udpa/annotations/security.upb.h +29 -4
- data/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.c +3 -3
- data/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.h +10 -2
- data/src/core/ext/upb-generated/udpa/annotations/status.upb.c +6 -6
- data/src/core/ext/upb-generated/udpa/annotations/status.upb.h +33 -8
- data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c +5 -5
- data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.h +26 -4
- data/src/core/ext/upb-generated/validate/validate.upb.c +264 -253
- data/src/core/ext/upb-generated/validate/validate.upb.h +1974 -709
- data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c +22 -22
- data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.h +115 -16
- data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c +6 -6
- data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.h +29 -4
- data/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.c +3 -3
- data/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.h +10 -2
- data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c +18 -18
- data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.h +94 -18
- data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c +5 -5
- data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.h +26 -4
- data/src/core/ext/upb-generated/xds/core/v3/authority.upb.c +2 -2
- data/src/core/ext/upb-generated/xds/core/v3/authority.upb.h +9 -2
- data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.c +47 -0
- data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.h +107 -0
- data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c +8 -8
- data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.h +50 -16
- data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c +5 -5
- data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.h +33 -12
- data/src/core/ext/upb-generated/xds/core/v3/extension.upb.c +3 -3
- data/src/core/ext/upb-generated/xds/core/v3/extension.upb.h +16 -4
- data/src/core/ext/upb-generated/xds/core/v3/resource.upb.c +5 -5
- data/src/core/ext/upb-generated/xds/core/v3/resource.upb.h +27 -10
- data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c +10 -10
- data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.h +83 -34
- data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c +5 -5
- data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.h +22 -4
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +14 -13
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +84 -28
- data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c +46 -0
- data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.h +113 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.c +46 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.h +98 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.c +62 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.h +174 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.c +36 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.h +74 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.c +65 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.h +191 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c +43 -39
- data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.h +309 -121
- data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.c +127 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.h +474 -0
- data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c +4 -4
- data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.h +31 -12
- data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c +9 -9
- data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.h +82 -29
- data/src/core/ext/upb-generated/xds/type/v3/cel.upb.c +67 -0
- data/src/core/ext/upb-generated/xds/type/v3/cel.upb.h +214 -0
- data/src/core/ext/upb-generated/xds/type/v3/range.upb.c +64 -0
- data/src/core/ext/upb-generated/xds/type/v3/range.upb.h +208 -0
- data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c +3 -4
- data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.h +16 -4
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/clusters.upbdefs.c +54 -53
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.c +67 -274
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.h +0 -85
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.c +277 -0
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.h +125 -0
- data/src/core/ext/upbdefs-generated/envoy/admin/v3/listeners.upbdefs.c +14 -11
- data/src/core/ext/upbdefs-generated/envoy/config/accesslog/v3/accesslog.upbdefs.c +107 -107
- data/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +199 -187
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/circuit_breaker.upbdefs.c +50 -46
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c +149 -129
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.h +0 -5
- data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/outlier_detection.upbdefs.c +16 -12
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.c +95 -75
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/base.upbdefs.c +187 -183
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.c +129 -94
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.h +10 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.c +18 -33
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.h +0 -5
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c +168 -154
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c +190 -186
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.c +7 -3
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/substitution_format_string.upbdefs.c +16 -16
- data/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint_components.upbdefs.c +71 -66
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c +164 -137
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener_components.upbdefs.c +141 -138
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c +16 -12
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/udp_listener_config.upbdefs.c +39 -31
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c +140 -129
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c +82 -68
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.h +2 -2
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +828 -747
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.h +20 -0
- data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.c +150 -139
- data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/datadog.upbdefs.c +16 -14
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c +48 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/trace.upbdefs.c +16 -12
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/zipkin.upbdefs.c +20 -18
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c +46 -25
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/router/v3/router.upbdefs.c +53 -42
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c +60 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +483 -445
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c +50 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c +21 -18
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +211 -193
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.c +156 -84
- data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c +113 -113
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.c +48 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/regex.upbdefs.c +29 -28
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.c +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.c +69 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_unit.upbdefs.c +10 -9
- data/src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c +92 -90
- data/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.c +99 -0
- data/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.h +75 -0
- data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.c +45 -0
- data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.c +43 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.c +51 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.c +36 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.h +35 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.c +55 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/matcher.upbdefs.c +9 -9
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.c +71 -0
- data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.h +60 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.c +60 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.h +40 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.c +36 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.h +45 -0
- data/src/core/ext/upbdefs-generated/xds/type/v3/typed_struct.upbdefs.c +4 -7
- data/src/core/ext/xds/certificate_provider_store.cc +75 -3
- data/src/core/ext/xds/certificate_provider_store.h +30 -4
- data/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +17 -10
- data/src/core/ext/xds/file_watcher_certificate_provider_factory.h +12 -4
- data/src/core/ext/xds/upb_utils.h +3 -25
- data/src/core/ext/xds/xds_api.cc +106 -192
- data/src/core/ext/xds/xds_api.h +41 -39
- data/src/core/ext/xds/xds_bootstrap.cc +5 -537
- data/src/core/ext/xds/xds_bootstrap.h +42 -95
- data/src/core/ext/xds/xds_bootstrap_grpc.cc +362 -0
- data/src/core/ext/xds/xds_bootstrap_grpc.h +184 -0
- data/src/core/ext/xds/xds_certificate_provider.cc +36 -25
- data/src/core/ext/xds/xds_certificate_provider.h +36 -4
- data/src/core/ext/xds/xds_channel_args.h +3 -3
- data/src/core/ext/xds/xds_channel_stack_modifier.cc +17 -7
- data/src/core/ext/xds/xds_channel_stack_modifier.h +16 -4
- data/src/core/ext/xds/xds_client.cc +885 -1360
- data/src/core/ext/xds/xds_client.h +69 -65
- data/src/core/ext/xds/xds_client_grpc.cc +235 -0
- data/src/core/ext/xds/xds_client_grpc.h +79 -0
- data/src/core/ext/xds/xds_client_stats.cc +24 -25
- data/src/core/ext/xds/xds_client_stats.h +26 -24
- data/src/core/ext/xds/xds_cluster.cc +505 -247
- data/src/core/ext/xds/xds_cluster.h +69 -40
- data/src/core/ext/xds/xds_cluster_specifier_plugin.cc +132 -0
- data/src/core/ext/xds/xds_cluster_specifier_plugin.h +97 -0
- data/src/core/ext/xds/xds_common_types.cc +238 -123
- data/src/core/ext/xds/xds_common_types.h +29 -16
- data/src/core/ext/xds/xds_endpoint.cc +266 -156
- data/src/core/ext/xds/xds_endpoint.h +19 -15
- data/src/core/ext/xds/xds_health_status.cc +80 -0
- data/src/core/ext/xds/xds_health_status.h +109 -0
- data/src/core/ext/xds/xds_http_fault_filter.cc +68 -59
- data/src/core/ext/xds/xds_http_fault_filter.h +19 -25
- data/src/core/ext/xds/xds_http_filters.cc +70 -71
- data/src/core/ext/xds/xds_http_filters.h +73 -25
- data/src/core/ext/xds/xds_http_rbac_filter.cc +168 -225
- data/src/core/ext/xds/xds_http_rbac_filter.h +22 -18
- data/src/core/ext/xds/xds_http_stateful_session_filter.cc +218 -0
- data/src/core/ext/xds/xds_http_stateful_session_filter.h +58 -0
- data/src/core/ext/xds/xds_lb_policy_registry.cc +334 -0
- data/src/core/ext/xds/xds_lb_policy_registry.h +71 -0
- data/src/core/ext/xds/xds_listener.cc +573 -481
- data/src/core/ext/xds/xds_listener.h +57 -51
- data/src/core/ext/xds/xds_resource_type.h +28 -22
- data/src/core/ext/xds/xds_resource_type_impl.h +18 -17
- data/src/core/ext/xds/xds_route_config.cc +638 -479
- data/src/core/ext/xds/xds_route_config.h +79 -42
- data/src/core/ext/xds/xds_routing.cc +21 -7
- data/src/core/ext/xds/xds_routing.h +17 -12
- data/src/core/ext/xds/xds_server_config_fetcher.cc +244 -200
- data/src/core/ext/xds/xds_transport.h +86 -0
- data/src/core/ext/xds/xds_transport_grpc.cc +356 -0
- data/src/core/ext/xds/xds_transport_grpc.h +135 -0
- data/src/core/lib/address_utils/parse_address.cc +52 -52
- data/src/core/lib/address_utils/parse_address.h +46 -42
- data/src/core/lib/address_utils/sockaddr_utils.cc +90 -54
- data/src/core/lib/address_utils/sockaddr_utils.h +45 -40
- data/src/core/lib/avl/avl.h +100 -13
- data/src/core/lib/backoff/backoff.cc +19 -21
- data/src/core/lib/backoff/backoff.h +21 -21
- data/src/core/lib/channel/call_finalization.h +6 -4
- data/src/core/lib/channel/call_tracer.h +27 -9
- data/src/core/lib/channel/channel_args.cc +268 -101
- data/src/core/lib/channel/channel_args.h +461 -56
- data/src/core/lib/channel/channel_args_preconditioning.cc +6 -11
- data/src/core/lib/channel/channel_args_preconditioning.h +7 -7
- data/src/core/lib/channel/channel_fwd.h +26 -0
- data/src/core/lib/channel/channel_stack.cc +92 -52
- data/src/core/lib/channel/channel_stack.h +186 -140
- data/src/core/lib/channel/channel_stack_builder.cc +26 -104
- data/src/core/lib/channel/channel_stack_builder.h +45 -31
- data/src/core/lib/channel/channel_stack_builder_impl.cc +126 -0
- data/src/core/lib/channel/channel_stack_builder_impl.h +48 -0
- data/src/core/lib/channel/channel_trace.cc +29 -36
- data/src/core/lib/channel/channel_trace.h +27 -24
- data/src/core/lib/channel/channelz.cc +50 -65
- data/src/core/lib/channel/channelz.h +50 -33
- data/src/core/lib/channel/channelz_registry.cc +28 -36
- data/src/core/lib/channel/channelz_registry.h +31 -30
- data/src/core/lib/channel/connected_channel.cc +1267 -78
- data/src/core/lib/channel/connected_channel.h +22 -23
- data/src/core/lib/channel/context.h +25 -21
- data/src/core/lib/channel/promise_based_filter.cc +2155 -303
- data/src/core/lib/channel/promise_based_filter.h +636 -122
- data/src/core/lib/channel/status_util.cc +64 -17
- data/src/core/lib/channel/status_util.h +38 -24
- data/src/core/lib/compression/compression.cc +24 -19
- data/src/core/lib/compression/compression_internal.cc +89 -70
- data/src/core/lib/compression/compression_internal.h +28 -27
- data/src/core/lib/compression/message_compress.cc +28 -26
- data/src/core/lib/compression/message_compress.h +28 -29
- data/src/core/lib/config/core_configuration.cc +8 -1
- data/src/core/lib/config/core_configuration.h +86 -39
- data/src/core/lib/debug/event_log.cc +88 -0
- data/src/core/lib/debug/event_log.h +81 -0
- data/src/core/lib/debug/histogram_view.cc +69 -0
- data/src/core/lib/debug/histogram_view.h +37 -0
- data/src/core/lib/debug/stats.cc +48 -152
- data/src/core/lib/debug/stats.h +50 -57
- data/src/core/lib/debug/stats_data.cc +302 -649
- data/src/core/lib/debug/stats_data.h +293 -548
- data/src/core/lib/debug/trace.cc +18 -20
- data/src/core/lib/debug/trace.h +27 -48
- data/src/core/lib/event_engine/channel_args_endpoint_config.cc +16 -17
- data/src/core/lib/event_engine/channel_args_endpoint_config.h +16 -9
- data/src/core/lib/event_engine/common_closures.h +71 -0
- data/src/core/lib/event_engine/default_event_engine.cc +99 -0
- data/src/core/lib/event_engine/default_event_engine.h +73 -0
- data/src/core/lib/event_engine/default_event_engine_factory.cc +24 -3
- data/src/core/lib/event_engine/{event_engine_factory.h → default_event_engine_factory.h} +6 -9
- data/src/core/lib/event_engine/event_engine.cc +3 -30
- data/src/core/lib/event_engine/executor/executor.h +38 -0
- data/src/core/lib/event_engine/forkable.cc +106 -0
- data/src/core/lib/event_engine/forkable.h +61 -0
- data/src/core/lib/event_engine/handle_containers.h +74 -0
- data/src/core/lib/event_engine/memory_allocator.cc +11 -3
- data/src/core/lib/event_engine/poller.h +62 -0
- data/src/core/lib/event_engine/posix.h +158 -0
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +642 -0
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.h +139 -0
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +899 -0
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.h +97 -0
- data/src/core/lib/event_engine/posix_engine/event_poller.h +111 -0
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +74 -0
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.h +33 -0
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +77 -0
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.h +179 -0
- data/src/core/lib/event_engine/posix_engine/lockfree_event.cc +265 -0
- data/src/core/lib/event_engine/posix_engine/lockfree_event.h +73 -0
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +1305 -0
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +717 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +640 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine.h +259 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_closure.h +80 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +289 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.h +279 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +379 -0
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.h +91 -0
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +853 -0
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +316 -0
- data/src/core/lib/event_engine/posix_engine/timer.cc +311 -0
- data/src/core/lib/event_engine/posix_engine/timer.h +194 -0
- data/src/core/lib/event_engine/posix_engine/timer_heap.cc +107 -0
- data/src/core/lib/event_engine/posix_engine/timer_heap.h +56 -0
- data/src/core/lib/event_engine/posix_engine/timer_manager.cc +173 -0
- data/src/core/lib/event_engine/posix_engine/timer_manager.h +114 -0
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +332 -0
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.h +185 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc +127 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.h +45 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc +150 -0
- data/src/core/lib/{iomgr/event_engine/promise.h → event_engine/posix_engine/wakeup_fd_pipe.h} +19 -25
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h +76 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc +67 -0
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h +37 -0
- data/src/core/lib/event_engine/resolved_address.cc +23 -2
- data/src/core/lib/{iomgr/event_engine → event_engine}/resolved_address_internal.h +5 -8
- data/src/core/lib/event_engine/shim.cc +56 -0
- data/src/core/lib/event_engine/shim.h +33 -0
- data/src/core/lib/event_engine/slice.cc +103 -0
- data/src/core/lib/event_engine/slice_buffer.cc +50 -0
- data/src/core/lib/event_engine/tcp_socket_utils.cc +389 -0
- data/src/core/lib/event_engine/tcp_socket_utils.h +90 -0
- data/src/core/lib/event_engine/thread_local.cc +29 -0
- data/src/core/lib/event_engine/thread_local.h +32 -0
- data/src/core/lib/event_engine/thread_pool.cc +253 -0
- data/src/core/lib/event_engine/thread_pool.h +141 -0
- data/src/core/lib/event_engine/time_util.cc +30 -0
- data/src/core/lib/event_engine/time_util.h +32 -0
- data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_event_engine.cc → lib/event_engine/trace.cc} +9 -13
- data/src/core/lib/event_engine/trace.h +43 -0
- data/src/core/lib/event_engine/utils.cc +44 -0
- data/src/core/lib/event_engine/utils.h +44 -0
- data/src/core/lib/event_engine/windows/iocp.cc +140 -0
- data/src/core/lib/event_engine/windows/iocp.h +69 -0
- data/src/core/lib/event_engine/windows/win_socket.cc +219 -0
- data/src/core/lib/event_engine/windows/win_socket.h +129 -0
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +331 -0
- data/src/core/lib/event_engine/windows/windows_endpoint.h +103 -0
- data/src/core/lib/event_engine/windows/windows_engine.cc +388 -0
- data/src/core/lib/event_engine/windows/windows_engine.h +163 -0
- data/src/core/lib/experiments/config.cc +161 -0
- data/src/core/lib/experiments/config.h +53 -0
- data/src/core/lib/experiments/experiments.cc +81 -0
- data/src/core/lib/experiments/experiments.h +117 -0
- data/src/core/lib/gpr/alloc.cc +19 -25
- data/src/core/lib/gpr/alloc.h +20 -20
- data/src/core/lib/gpr/atm.cc +17 -17
- data/src/core/lib/gpr/cpu_iphone.cc +24 -24
- data/src/core/lib/gpr/cpu_linux.cc +28 -23
- data/src/core/lib/gpr/cpu_posix.cc +23 -22
- data/src/core/lib/gpr/cpu_windows.cc +20 -18
- data/src/core/lib/gpr/log.cc +30 -17
- data/src/core/lib/gpr/log_android.cc +22 -20
- data/src/core/lib/gpr/log_linux.cc +24 -24
- data/src/core/lib/gpr/log_posix.cc +20 -19
- data/src/core/lib/gpr/log_windows.cc +25 -25
- data/src/core/lib/gpr/spinlock.h +20 -20
- data/src/core/lib/gpr/string.cc +25 -24
- data/src/core/lib/gpr/string.h +61 -61
- data/src/core/lib/gpr/string_posix.cc +24 -24
- data/src/core/lib/gpr/string_util_windows.cc +25 -52
- data/src/core/lib/gpr/string_windows.cc +24 -24
- data/src/core/lib/gpr/sync.cc +25 -25
- data/src/core/lib/gpr/sync_abseil.cc +36 -40
- data/src/core/lib/gpr/sync_posix.cc +22 -34
- data/src/core/lib/gpr/sync_windows.cc +29 -27
- data/src/core/lib/gpr/time.cc +34 -30
- data/src/core/lib/gpr/time_posix.cc +41 -45
- data/src/core/lib/gpr/time_precise.cc +22 -22
- data/src/core/lib/gpr/time_precise.h +21 -22
- data/src/core/lib/gpr/time_windows.cc +35 -29
- data/src/core/lib/gpr/tmpfile.h +24 -24
- data/src/core/lib/gpr/tmpfile_msys.cc +21 -20
- data/src/core/lib/gpr/tmpfile_posix.cc +22 -20
- data/src/core/lib/gpr/tmpfile_windows.cc +28 -29
- data/src/core/lib/gpr/useful.h +83 -30
- data/src/core/lib/gpr/wrap_memcpy.cc +23 -23
- data/src/core/lib/gprpp/atomic_utils.h +20 -20
- data/src/core/lib/gprpp/bitset.h +47 -17
- data/src/core/lib/gprpp/chunked_vector.h +7 -3
- data/src/core/lib/gprpp/construct_destruct.h +4 -3
- data/src/core/lib/gprpp/cpp_impl_of.h +7 -3
- data/src/core/{ext/xds/xds_resource_type.cc → lib/gprpp/crash.cc} +12 -12
- data/src/core/lib/gprpp/crash.h +34 -0
- data/src/core/lib/gprpp/debug_location.h +60 -31
- data/src/core/lib/gprpp/dual_ref_counted.h +4 -7
- data/src/core/lib/gprpp/env.h +53 -0
- data/src/core/lib/gprpp/env_linux.cc +80 -0
- data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_event_engine.cc → lib/gprpp/env_posix.cc} +25 -9
- data/src/core/lib/gprpp/env_windows.cc +56 -0
- data/src/core/lib/gprpp/examine_stack.cc +17 -17
- data/src/core/lib/gprpp/examine_stack.h +21 -22
- data/src/core/lib/gprpp/fork.cc +58 -53
- data/src/core/lib/gprpp/fork.h +29 -35
- data/src/core/lib/gprpp/global_config.h +22 -24
- data/src/core/lib/gprpp/global_config_custom.h +20 -20
- data/src/core/lib/gprpp/global_config_env.cc +28 -25
- data/src/core/lib/gprpp/global_config_env.h +29 -27
- data/src/core/lib/gprpp/global_config_generic.h +21 -25
- data/src/core/lib/gprpp/host_port.cc +30 -26
- data/src/core/lib/gprpp/host_port.h +32 -31
- data/src/core/lib/gprpp/load_file.cc +75 -0
- data/src/core/lib/gprpp/load_file.h +33 -0
- data/src/core/lib/gprpp/manual_constructor.h +21 -89
- data/src/core/lib/gprpp/match.h +75 -0
- data/src/core/lib/gprpp/memory.h +22 -26
- data/src/core/lib/gprpp/mpscq.cc +17 -17
- data/src/core/lib/gprpp/mpscq.h +21 -21
- data/src/core/lib/gprpp/no_destruct.h +95 -0
- data/src/core/lib/gprpp/notification.h +67 -0
- data/src/core/lib/gprpp/orphanable.h +22 -25
- data/src/core/lib/gprpp/overload.h +59 -0
- data/src/core/lib/gprpp/packed_table.h +40 -0
- data/src/core/lib/gprpp/per_cpu.h +46 -0
- data/src/core/lib/gprpp/ref_counted.h +24 -24
- data/src/core/lib/gprpp/ref_counted_ptr.h +42 -55
- data/src/core/lib/gprpp/single_set_ptr.h +87 -0
- data/src/core/lib/gprpp/sorted_pack.h +98 -0
- data/src/core/lib/gprpp/stat.h +3 -5
- data/src/core/lib/gprpp/stat_posix.cc +10 -4
- data/src/core/lib/gprpp/stat_windows.cc +4 -2
- data/src/core/lib/gprpp/status_helper.cc +56 -20
- data/src/core/lib/gprpp/status_helper.h +13 -5
- data/src/core/lib/gprpp/strerror.cc +43 -0
- data/src/core/lib/gprpp/strerror.h +29 -0
- data/src/core/lib/gprpp/sync.h +26 -24
- data/src/core/lib/gprpp/table.h +22 -4
- data/src/core/lib/gprpp/tchar.cc +49 -0
- data/src/core/lib/gprpp/tchar.h +33 -0
- data/src/core/lib/gprpp/thd.h +25 -28
- data/src/core/lib/gprpp/thd_posix.cc +33 -32
- data/src/core/lib/gprpp/thd_windows.cc +28 -26
- data/src/core/lib/gprpp/time.cc +64 -9
- data/src/core/lib/gprpp/time.h +79 -5
- data/src/core/lib/gprpp/time_averaged_stats.cc +60 -0
- data/src/core/lib/gprpp/time_averaged_stats.h +79 -0
- data/src/core/lib/gprpp/time_util.cc +4 -0
- data/src/core/lib/gprpp/time_util.h +3 -3
- data/src/core/lib/gprpp/unique_type_name.h +104 -0
- data/src/core/lib/gprpp/validation_errors.cc +61 -0
- data/src/core/lib/gprpp/validation_errors.h +127 -0
- data/src/core/lib/{iomgr → gprpp}/work_serializer.cc +34 -18
- data/src/core/lib/{iomgr → gprpp}/work_serializer.h +21 -28
- data/src/core/lib/handshaker/proxy_mapper.h +53 -0
- data/src/core/lib/handshaker/proxy_mapper_registry.cc +71 -0
- data/src/core/lib/handshaker/proxy_mapper_registry.h +75 -0
- data/src/core/lib/http/format_request.cc +54 -24
- data/src/core/lib/http/format_request.h +23 -21
- data/src/core/lib/http/httpcli.cc +139 -141
- data/src/core/lib/http/httpcli.h +86 -38
- data/src/core/lib/http/httpcli_security_connector.cc +53 -45
- data/src/core/lib/http/httpcli_ssl_credentials.h +6 -4
- data/src/core/lib/http/parser.cc +144 -85
- data/src/core/lib/http/parser.h +51 -35
- data/src/core/lib/iomgr/block_annotate.h +23 -23
- data/src/core/lib/iomgr/buffer_list.cc +156 -136
- data/src/core/lib/iomgr/buffer_list.h +123 -101
- data/src/core/lib/iomgr/call_combiner.cc +32 -64
- data/src/core/lib/iomgr/call_combiner.h +24 -25
- data/src/core/lib/iomgr/cfstream_handle.cc +33 -36
- data/src/core/lib/iomgr/cfstream_handle.h +25 -25
- data/src/core/lib/iomgr/{event_engine/pollset.h → closure.cc} +9 -7
- data/src/core/lib/iomgr/closure.h +95 -58
- data/src/core/lib/iomgr/combiner.cc +20 -39
- data/src/core/lib/iomgr/combiner.h +20 -20
- data/src/core/lib/iomgr/dualstack_socket_posix.cc +21 -21
- data/src/core/lib/iomgr/dynamic_annotations.h +22 -22
- data/src/core/lib/iomgr/endpoint.cc +21 -21
- data/src/core/lib/iomgr/endpoint.h +53 -51
- data/src/core/lib/iomgr/endpoint_cfstream.cc +51 -47
- data/src/core/lib/iomgr/endpoint_cfstream.h +32 -32
- data/src/core/lib/iomgr/endpoint_pair.h +20 -20
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +34 -27
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +30 -21
- data/src/core/lib/iomgr/error.cc +51 -834
- data/src/core/lib/iomgr/error.h +45 -317
- data/src/core/lib/iomgr/error_cfstream.cc +18 -23
- data/src/core/lib/iomgr/error_cfstream.h +21 -21
- data/src/core/lib/iomgr/ev_apple.cc +21 -21
- data/src/core/lib/iomgr/ev_apple.h +21 -21
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +263 -248
- data/src/core/lib/iomgr/ev_epoll1_linux.h +21 -21
- data/src/core/lib/iomgr/ev_poll_posix.cc +296 -271
- data/src/core/lib/iomgr/ev_poll_posix.h +22 -22
- data/src/core/lib/iomgr/ev_posix.cc +88 -122
- data/src/core/lib/iomgr/ev_posix.h +93 -87
- data/src/core/lib/iomgr/ev_windows.cc +18 -18
- data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.cc +15 -30
- data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.h +10 -13
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +430 -0
- data/src/core/lib/iomgr/event_engine_shims/endpoint.h +43 -0
- data/src/core/lib/iomgr/event_engine_shims/tcp_client.cc +91 -0
- data/src/core/lib/iomgr/event_engine_shims/tcp_client.h +44 -0
- data/src/core/lib/iomgr/exec_ctx.cc +34 -56
- data/src/core/lib/iomgr/exec_ctx.h +152 -177
- data/src/core/lib/iomgr/executor.cc +21 -31
- data/src/core/lib/iomgr/executor.h +27 -30
- data/src/core/lib/iomgr/fork_posix.cc +31 -28
- data/src/core/lib/iomgr/fork_windows.cc +21 -21
- data/src/core/lib/iomgr/gethostname.h +20 -20
- data/src/core/lib/iomgr/gethostname_fallback.cc +17 -17
- data/src/core/lib/iomgr/gethostname_host_name_max.cc +17 -17
- data/src/core/lib/iomgr/gethostname_sysconf.cc +17 -17
- data/src/core/lib/iomgr/grpc_if_nametoindex.h +22 -22
- data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +20 -19
- data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +20 -19
- data/src/core/lib/iomgr/internal_errqueue.cc +41 -48
- data/src/core/lib/iomgr/internal_errqueue.h +84 -89
- data/src/core/lib/iomgr/iocp_windows.cc +23 -23
- data/src/core/lib/iomgr/iocp_windows.h +21 -21
- data/src/core/lib/iomgr/iomgr.cc +25 -20
- data/src/core/lib/iomgr/iomgr.h +35 -35
- data/src/core/lib/iomgr/iomgr_fwd.h +26 -0
- data/src/core/lib/iomgr/iomgr_internal.cc +17 -17
- data/src/core/lib/iomgr/iomgr_internal.h +28 -28
- data/src/core/lib/iomgr/iomgr_posix.cc +21 -20
- data/src/core/lib/iomgr/iomgr_posix_cfstream.cc +21 -19
- data/src/core/lib/iomgr/iomgr_windows.cc +24 -22
- data/src/core/lib/iomgr/load_file.cc +24 -27
- data/src/core/lib/iomgr/load_file.h +22 -22
- data/src/core/lib/iomgr/lockfree_event.cc +114 -131
- data/src/core/lib/iomgr/lockfree_event.h +23 -23
- data/src/core/lib/iomgr/nameser.h +86 -86
- data/src/core/lib/iomgr/polling_entity.cc +25 -21
- data/src/core/lib/iomgr/polling_entity.h +29 -29
- data/src/core/lib/iomgr/pollset.cc +17 -17
- data/src/core/lib/iomgr/pollset.h +55 -55
- data/src/core/lib/iomgr/pollset_set.cc +17 -17
- data/src/core/lib/iomgr/pollset_set.h +26 -28
- data/src/core/lib/iomgr/pollset_set_windows.cc +18 -18
- data/src/core/lib/iomgr/pollset_set_windows.h +20 -20
- data/src/core/lib/iomgr/pollset_windows.cc +32 -31
- data/src/core/lib/iomgr/pollset_windows.h +24 -24
- data/src/core/lib/iomgr/port.h +59 -39
- data/src/core/lib/iomgr/python_util.h +24 -24
- data/src/core/lib/iomgr/resolve_address.cc +34 -20
- data/src/core/lib/iomgr/resolve_address.h +73 -43
- data/src/core/lib/iomgr/resolve_address_impl.h +7 -7
- data/src/core/lib/iomgr/resolve_address_posix.cc +81 -62
- data/src/core/lib/iomgr/resolve_address_posix.h +26 -10
- data/src/core/lib/iomgr/resolve_address_windows.cc +64 -38
- data/src/core/lib/iomgr/resolve_address_windows.h +26 -10
- data/src/core/lib/iomgr/resolved_address.h +3 -3
- data/src/core/lib/iomgr/sockaddr.h +25 -26
- data/src/core/lib/iomgr/sockaddr_posix.h +23 -21
- data/src/core/lib/iomgr/sockaddr_utils_posix.cc +20 -18
- data/src/core/lib/iomgr/sockaddr_windows.h +23 -21
- data/src/core/lib/iomgr/socket_factory_posix.cc +18 -18
- data/src/core/lib/iomgr/socket_factory_posix.h +32 -32
- data/src/core/lib/iomgr/socket_mutator.cc +19 -18
- data/src/core/lib/iomgr/socket_mutator.h +39 -39
- data/src/core/lib/iomgr/socket_utils.h +27 -27
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +82 -101
- data/src/core/lib/iomgr/socket_utils_linux.cc +18 -17
- data/src/core/lib/iomgr/socket_utils_posix.cc +103 -19
- data/src/core/lib/iomgr/socket_utils_posix.h +176 -84
- data/src/core/lib/iomgr/socket_utils_windows.cc +20 -19
- data/src/core/lib/iomgr/socket_windows.cc +37 -36
- data/src/core/lib/iomgr/socket_windows.h +59 -61
- data/src/core/lib/iomgr/systemd_utils.cc +116 -0
- data/src/core/lib/iomgr/systemd_utils.h +33 -0
- data/src/core/lib/iomgr/tcp_client.cc +28 -24
- data/src/core/lib/iomgr/tcp_client.h +49 -38
- data/src/core/lib/iomgr/tcp_client_cfstream.cc +46 -35
- data/src/core/lib/iomgr/tcp_client_posix.cc +244 -106
- data/src/core/lib/iomgr/tcp_client_posix.h +55 -52
- data/src/core/lib/iomgr/tcp_client_windows.cc +73 -53
- data/src/core/lib/iomgr/tcp_posix.cc +562 -361
- data/src/core/lib/iomgr/tcp_posix.h +32 -30
- data/src/core/lib/iomgr/tcp_server.cc +33 -24
- data/src/core/lib/iomgr/tcp_server.h +78 -69
- data/src/core/lib/iomgr/tcp_server_posix.cc +402 -145
- data/src/core/lib/iomgr/tcp_server_utils_posix.h +68 -55
- data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +120 -82
- data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +46 -43
- data/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc +19 -19
- data/src/core/lib/iomgr/tcp_server_windows.cc +116 -118
- data/src/core/lib/iomgr/tcp_windows.cc +116 -98
- data/src/core/lib/iomgr/tcp_windows.h +34 -35
- data/src/core/lib/iomgr/timer.cc +17 -17
- data/src/core/lib/iomgr/timer.h +68 -68
- data/src/core/lib/iomgr/timer_generic.cc +125 -134
- data/src/core/lib/iomgr/timer_generic.h +21 -21
- data/src/core/lib/iomgr/timer_heap.cc +25 -25
- data/src/core/lib/iomgr/timer_heap.h +22 -22
- data/src/core/lib/iomgr/timer_manager.cc +31 -31
- data/src/core/lib/iomgr/timer_manager.h +27 -27
- data/src/core/lib/iomgr/unix_sockets_posix.cc +20 -21
- data/src/core/lib/iomgr/unix_sockets_posix.h +21 -21
- data/src/core/lib/iomgr/unix_sockets_posix_noop.cc +19 -17
- data/src/core/lib/iomgr/wakeup_fd_eventfd.cc +23 -23
- data/src/core/lib/iomgr/wakeup_fd_nospecial.cc +23 -23
- data/src/core/lib/iomgr/wakeup_fd_pipe.cc +29 -26
- data/src/core/lib/iomgr/wakeup_fd_pipe.h +20 -20
- data/src/core/lib/iomgr/wakeup_fd_posix.cc +33 -30
- data/src/core/lib/iomgr/wakeup_fd_posix.h +52 -54
- data/src/core/lib/json/json.h +23 -27
- data/src/core/{ext/filters/http/message_compress/message_decompress_filter.h → lib/json/json_args.h} +13 -10
- data/src/core/lib/json/json_channel_args.h +42 -0
- data/src/core/lib/json/json_object_loader.cc +217 -0
- data/src/core/lib/json/json_object_loader.h +634 -0
- data/src/core/lib/json/json_reader.cc +123 -90
- data/src/core/lib/json/json_util.cc +14 -34
- data/src/core/lib/json/json_util.h +19 -10
- data/src/core/lib/json/json_writer.cc +62 -57
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.cc +30 -60
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.h +47 -62
- data/src/core/lib/load_balancing/lb_policy_factory.h +49 -0
- data/src/core/lib/load_balancing/lb_policy_registry.cc +142 -0
- data/src/core/lib/load_balancing/lb_policy_registry.h +82 -0
- data/src/core/{ext/filters/client_channel → lib/load_balancing}/subchannel_interface.h +57 -52
- data/src/core/lib/matchers/matchers.cc +12 -7
- data/src/core/lib/matchers/matchers.h +5 -3
- data/src/core/lib/promise/activity.cc +23 -8
- data/src/core/lib/promise/activity.h +116 -58
- data/src/core/lib/promise/arena_promise.h +115 -72
- data/src/core/lib/promise/context.h +18 -11
- data/src/core/lib/promise/detail/basic_join.h +197 -0
- data/src/core/lib/promise/detail/basic_seq.h +28 -33
- data/src/core/lib/promise/detail/promise_factory.h +61 -14
- data/src/core/lib/promise/detail/promise_like.h +3 -3
- data/src/core/lib/promise/detail/status.h +33 -4
- data/src/core/lib/promise/detail/switch.h +21 -21
- data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h +23 -15
- data/src/core/lib/promise/if.h +195 -0
- data/src/core/lib/promise/interceptor_list.h +308 -0
- data/src/core/lib/promise/intra_activity_waiter.h +55 -0
- data/src/core/lib/promise/latch.h +191 -0
- data/src/core/lib/promise/loop.h +13 -9
- data/src/core/lib/promise/map.h +4 -6
- data/src/core/lib/promise/pipe.h +608 -0
- data/src/core/lib/promise/poll.h +129 -11
- data/src/core/lib/promise/promise.h +5 -5
- data/src/core/lib/promise/race.h +6 -10
- data/src/core/lib/promise/seq.h +32 -14
- data/src/core/lib/promise/sleep.cc +58 -42
- data/src/core/lib/promise/sleep.h +44 -26
- data/src/core/lib/promise/trace.cc +20 -0
- data/src/core/{ext/filters/max_age/max_age_filter.h → lib/promise/trace.h} +6 -8
- data/src/core/lib/promise/try_join.h +82 -0
- data/src/core/lib/promise/try_seq.h +41 -23
- data/src/core/lib/resolver/resolver.cc +17 -59
- data/src/core/lib/resolver/resolver.h +21 -18
- data/src/core/lib/resolver/resolver_factory.h +11 -10
- data/src/core/lib/resolver/resolver_registry.cc +17 -9
- data/src/core/lib/resolver/resolver_registry.h +15 -5
- data/src/core/lib/resolver/server_address.cc +46 -34
- data/src/core/lib/resolver/server_address.h +36 -35
- data/src/core/lib/resource_quota/api.cc +25 -29
- data/src/core/lib/resource_quota/api.h +14 -5
- data/src/core/lib/resource_quota/arena.cc +85 -27
- data/src/core/lib/resource_quota/arena.h +197 -25
- data/src/core/lib/resource_quota/memory_quota.cc +322 -101
- data/src/core/lib/resource_quota/memory_quota.h +191 -56
- data/src/core/lib/resource_quota/periodic_update.cc +78 -0
- data/src/core/lib/resource_quota/periodic_update.h +71 -0
- data/src/core/lib/resource_quota/resource_quota.h +20 -4
- data/src/core/lib/resource_quota/thread_quota.cc +2 -0
- data/src/core/lib/resource_quota/thread_quota.h +7 -3
- data/src/core/lib/resource_quota/trace.h +3 -3
- data/src/core/lib/security/authorization/authorization_engine.h +3 -3
- data/src/core/lib/security/authorization/authorization_policy_provider.h +17 -3
- data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +3 -0
- data/src/core/lib/security/authorization/evaluate_args.cc +17 -8
- data/src/core/lib/security/authorization/evaluate_args.h +9 -6
- data/src/core/lib/security/authorization/grpc_authorization_engine.cc +5 -1
- data/src/core/lib/security/authorization/grpc_authorization_engine.h +10 -3
- data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +26 -13
- data/src/core/lib/security/authorization/grpc_server_authz_filter.h +20 -10
- data/src/core/lib/security/authorization/matchers.cc +44 -30
- data/src/core/lib/security/authorization/matchers.h +10 -3
- data/src/core/lib/security/authorization/rbac_policy.cc +6 -2
- data/src/core/lib/security/authorization/rbac_policy.h +10 -3
- data/src/core/{ext/xds → lib/security/certificate_provider}/certificate_provider_factory.h +9 -4
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +60 -0
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.h +70 -0
- data/src/core/lib/security/context/security_context.cc +27 -25
- data/src/core/lib/security/context/security_context.h +64 -35
- data/src/core/lib/security/credentials/alts/alts_credentials.cc +35 -27
- data/src/core/lib/security/credentials/alts/alts_credentials.h +65 -57
- data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +19 -19
- data/src/core/lib/security/credentials/alts/check_gcp_environment.h +43 -43
- data/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +17 -17
- data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +18 -17
- data/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc +18 -17
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +20 -22
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +18 -18
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h +38 -39
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc +19 -22
- data/src/core/lib/security/credentials/call_creds_util.cc +13 -3
- data/src/core/lib/security/credentials/call_creds_util.h +6 -5
- data/src/core/lib/security/credentials/channel_creds_registry.h +10 -4
- data/src/core/lib/security/credentials/channel_creds_registry_init.cc +13 -3
- data/src/core/lib/security/credentials/composite/composite_credentials.cc +46 -41
- data/src/core/lib/security/credentials/composite/composite_credentials.h +51 -38
- data/src/core/lib/security/credentials/credentials.cc +23 -26
- data/src/core/lib/security/credentials/credentials.h +91 -94
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +202 -74
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +21 -3
- data/src/core/lib/security/credentials/external/aws_request_signer.cc +14 -3
- data/src/core/lib/security/credentials/external/aws_request_signer.h +3 -3
- data/src/core/lib/security/credentials/external/external_account_credentials.cc +86 -68
- data/src/core/lib/security/credentials/external/external_account_credentials.h +14 -3
- data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +29 -26
- data/src/core/lib/security/credentials/external/file_external_account_credentials.h +9 -3
- data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +44 -31
- data/src/core/lib/security/credentials/external/url_external_account_credentials.h +13 -3
- data/src/core/lib/security/credentials/fake/fake_credentials.cc +48 -48
- data/src/core/lib/security/credentials/fake/fake_credentials.h +57 -47
- data/src/core/lib/security/credentials/google_default/credentials_generic.cc +23 -25
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +148 -112
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +35 -29
- data/src/core/lib/security/credentials/iam/iam_credentials.cc +35 -24
- data/src/core/lib/security/credentials/iam/iam_credentials.h +37 -23
- data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +32 -29
- data/src/core/lib/security/credentials/insecure/insecure_credentials.h +61 -0
- data/src/core/lib/security/credentials/jwt/json_token.cc +40 -29
- data/src/core/lib/security/credentials/jwt/json_token.h +36 -36
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +37 -30
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +41 -23
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +113 -89
- data/src/core/lib/security/credentials/jwt/jwt_verifier.h +49 -47
- data/src/core/lib/security/credentials/local/local_credentials.cc +36 -30
- data/src/core/lib/security/credentials/local/local_credentials.h +36 -26
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +107 -91
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +56 -27
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +42 -36
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +55 -29
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +48 -44
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +43 -23
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc +22 -30
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h +16 -17
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +65 -48
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +69 -9
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +44 -4
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +75 -13
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +22 -23
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +69 -98
- data/src/core/lib/security/credentials/tls/tls_credentials.cc +55 -50
- data/src/core/lib/security/credentials/tls/tls_credentials.h +33 -30
- data/src/core/lib/security/credentials/tls/tls_utils.cc +5 -1
- data/src/core/lib/security/credentials/tls/tls_utils.h +4 -4
- data/src/core/lib/security/credentials/xds/xds_credentials.cc +77 -81
- data/src/core/lib/security/credentials/xds/xds_credentials.h +50 -10
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +60 -54
- data/src/core/lib/security/security_connector/alts/alts_security_connector.h +49 -47
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +75 -79
- data/src/core/lib/security/security_connector/fake/fake_security_connector.h +25 -26
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc +15 -6
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.h +26 -15
- data/src/core/lib/security/security_connector/load_system_roots.h +20 -20
- data/src/core/lib/security/security_connector/load_system_roots_fallback.cc +22 -20
- data/src/core/lib/security/security_connector/{load_system_roots_linux.cc → load_system_roots_supported.cc} +43 -38
- data/src/core/lib/security/security_connector/load_system_roots_supported.h +45 -0
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +66 -49
- data/src/core/lib/security/security_connector/local/local_security_connector.h +49 -46
- data/src/core/lib/security/security_connector/security_connector.cc +37 -35
- data/src/core/lib/security/security_connector/security_connector.h +67 -45
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +59 -48
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +44 -43
- data/src/core/lib/security/security_connector/ssl_utils.cc +47 -39
- data/src/core/lib/security/security_connector/ssl_utils.h +45 -42
- data/src/core/lib/security/security_connector/ssl_utils_config.cc +22 -22
- data/src/core/lib/security/security_connector/ssl_utils_config.h +21 -22
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +62 -58
- data/src/core/lib/security/security_connector/tls/tls_security_connector.h +48 -28
- data/src/core/lib/security/transport/auth_filters.h +59 -30
- data/src/core/lib/security/transport/client_auth_filter.cc +73 -48
- data/src/core/lib/security/transport/secure_endpoint.cc +305 -181
- data/src/core/lib/security/transport/secure_endpoint.h +29 -28
- data/src/core/lib/security/transport/security_handshaker.cc +142 -110
- data/src/core/lib/security/transport/security_handshaker.h +28 -23
- data/src/core/lib/security/transport/server_auth_filter.cc +157 -266
- data/src/core/lib/security/transport/tsi_error.cc +23 -20
- data/src/core/lib/security/transport/tsi_error.h +20 -20
- data/src/core/lib/security/util/json_util.cc +24 -24
- data/src/core/lib/security/util/json_util.h +21 -23
- data/src/core/lib/service_config/service_config.h +19 -12
- data/src/core/lib/service_config/service_config_call_data.h +15 -5
- data/src/core/lib/service_config/service_config_impl.cc +111 -150
- data/src/core/lib/service_config/service_config_impl.h +27 -27
- data/src/core/lib/service_config/service_config_parser.cc +16 -28
- data/src/core/lib/service_config/service_config_parser.h +18 -19
- data/src/core/lib/slice/b64.cc +26 -26
- data/src/core/lib/slice/b64.h +34 -32
- data/src/core/lib/slice/percent_encoding.cc +24 -30
- data/src/core/lib/slice/percent_encoding.h +28 -34
- data/src/core/lib/slice/slice.cc +59 -46
- data/src/core/lib/slice/slice.h +64 -20
- data/src/core/lib/slice/slice_buffer.cc +136 -76
- data/src/core/lib/slice/slice_buffer.h +168 -0
- data/src/core/lib/slice/slice_internal.h +38 -45
- data/src/core/lib/slice/slice_refcount.cc +3 -18
- data/src/core/lib/slice/slice_refcount.h +53 -19
- data/src/core/lib/slice/slice_string_helpers.cc +17 -33
- data/src/core/lib/slice/slice_string_helpers.h +23 -30
- data/src/core/lib/surface/api_trace.cc +17 -17
- data/src/core/lib/surface/api_trace.h +25 -25
- data/src/core/lib/surface/builtins.cc +7 -2
- data/src/core/lib/surface/builtins.h +3 -3
- data/src/core/lib/surface/byte_buffer.cc +26 -21
- data/src/core/lib/surface/byte_buffer_reader.cc +24 -24
- data/src/core/lib/surface/call.cc +3000 -1131
- data/src/core/lib/surface/call.h +135 -56
- data/src/core/lib/surface/call_details.cc +22 -23
- data/src/core/lib/surface/call_log_batch.cc +25 -18
- data/src/core/lib/surface/call_test_only.h +34 -31
- data/src/core/lib/surface/call_trace.cc +123 -0
- data/src/core/lib/surface/call_trace.h +30 -0
- data/src/core/lib/surface/channel.cc +218 -284
- data/src/core/lib/surface/channel.h +137 -90
- data/src/core/lib/surface/channel_init.cc +19 -20
- data/src/core/lib/surface/channel_init.h +24 -26
- data/src/core/lib/surface/channel_ping.cc +26 -20
- data/src/core/lib/surface/channel_stack_type.cc +21 -19
- data/src/core/lib/surface/channel_stack_type.h +22 -22
- data/src/core/lib/surface/completion_queue.cc +200 -222
- data/src/core/lib/surface/completion_queue.h +42 -40
- data/src/core/lib/surface/completion_queue_factory.cc +34 -28
- data/src/core/lib/surface/completion_queue_factory.h +21 -23
- data/src/core/lib/surface/event_string.cc +19 -24
- data/src/core/lib/surface/event_string.h +21 -21
- data/src/core/lib/surface/init.cc +57 -114
- data/src/core/lib/surface/init.h +20 -28
- data/src/core/lib/surface/init_internally.cc +25 -0
- data/src/core/lib/surface/init_internally.h +37 -0
- data/src/core/lib/surface/lame_client.cc +87 -130
- data/src/core/lib/surface/lame_client.h +62 -24
- data/src/core/lib/surface/metadata_array.cc +18 -17
- data/src/core/lib/surface/server.cc +409 -188
- data/src/core/lib/surface/server.h +57 -26
- data/src/core/lib/surface/validate_metadata.cc +34 -48
- data/src/core/lib/surface/validate_metadata.h +24 -21
- data/src/core/lib/surface/version.cc +21 -21
- data/src/core/lib/transport/bdp_estimator.cc +19 -19
- data/src/core/lib/transport/bdp_estimator.h +22 -24
- data/src/core/lib/transport/connectivity_state.cc +24 -23
- data/src/core/lib/transport/connectivity_state.h +24 -25
- data/src/core/lib/transport/error_utils.cc +57 -79
- data/src/core/lib/transport/error_utils.h +29 -25
- data/src/core/lib/{channel → transport}/handshaker.cc +75 -58
- data/src/core/lib/{channel → transport}/handshaker.h +59 -48
- data/src/core/lib/transport/handshaker_factory.h +74 -0
- data/src/core/lib/transport/handshaker_registry.cc +61 -0
- data/src/core/lib/transport/handshaker_registry.h +69 -0
- data/src/core/lib/transport/http2_errors.h +22 -22
- data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.cc +78 -65
- data/src/core/lib/transport/http_connect_handshaker.h +42 -0
- data/src/core/lib/transport/metadata_batch.cc +305 -0
- data/src/core/lib/transport/metadata_batch.h +269 -302
- data/src/core/lib/transport/parsed_metadata.cc +2 -4
- data/src/core/lib/transport/parsed_metadata.h +17 -8
- data/src/core/lib/transport/pid_controller.cc +20 -20
- data/src/core/lib/transport/pid_controller.h +27 -27
- data/src/core/lib/transport/status_conversion.cc +23 -23
- data/src/core/lib/transport/status_conversion.h +24 -24
- data/src/core/lib/transport/tcp_connect_handshaker.cc +245 -0
- data/src/core/lib/transport/tcp_connect_handshaker.h +39 -0
- data/src/core/lib/transport/timeout_encoding.cc +24 -28
- data/src/core/lib/transport/timeout_encoding.h +25 -21
- data/src/core/lib/transport/transport.cc +97 -57
- data/src/core/lib/transport/transport.h +252 -207
- data/src/core/lib/transport/transport_fwd.h +20 -0
- data/src/core/lib/transport/transport_impl.h +59 -48
- data/src/core/lib/transport/transport_op_string.cc +40 -39
- data/src/core/lib/uri/uri_parser.cc +12 -4
- data/src/core/lib/uri/uri_parser.h +3 -5
- data/src/core/plugin_registry/grpc_plugin_registry.cc +62 -73
- data/src/core/plugin_registry/grpc_plugin_registry_extra.cc +20 -39
- data/src/core/tsi/alts/crypt/aes_gcm.cc +20 -20
- data/src/core/tsi/alts/crypt/gsec.cc +26 -26
- data/src/core/tsi/alts/crypt/gsec.h +336 -336
- data/src/core/tsi/alts/frame_protector/alts_counter.cc +23 -23
- data/src/core/tsi/alts/frame_protector/alts_counter.h +68 -68
- data/src/core/tsi/alts/frame_protector/alts_crypter.cc +19 -19
- data/src/core/tsi/alts/frame_protector/alts_crypter.h +209 -209
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +72 -71
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.h +40 -40
- data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc +18 -18
- data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h +82 -83
- data/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc +22 -22
- data/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc +22 -22
- data/src/core/tsi/alts/frame_protector/frame_handler.cc +26 -25
- data/src/core/tsi/alts/frame_protector/frame_handler.h +169 -169
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +117 -97
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +104 -104
- data/src/core/tsi/alts/handshaker/alts_shared_resource.cc +18 -17
- data/src/core/tsi/alts/handshaker/alts_shared_resource.h +43 -44
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +53 -45
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +60 -60
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h +24 -24
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +21 -20
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.h +38 -38
- data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +19 -19
- data/src/core/tsi/alts/handshaker/transport_security_common_api.h +105 -104
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +45 -44
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h +40 -41
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +38 -36
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h +35 -36
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol.h +67 -68
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +32 -31
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.h +55 -56
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc +54 -53
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h +141 -142
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +67 -58
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h +43 -44
- data/src/core/tsi/fake_transport_security.cc +146 -110
- data/src/core/tsi/fake_transport_security.h +36 -30
- data/src/core/tsi/local_transport_security.cc +43 -38
- data/src/core/tsi/local_transport_security.h +33 -33
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.cc +9 -4
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.h +3 -3
- data/src/core/tsi/ssl/session_cache/ssl_session.h +23 -21
- data/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc +21 -19
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +18 -17
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.h +31 -22
- data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +27 -20
- data/src/core/tsi/ssl_transport_security.cc +356 -390
- data/src/core/tsi/ssl_transport_security.h +210 -201
- data/src/core/tsi/ssl_transport_security_utils.cc +250 -0
- data/src/core/tsi/ssl_transport_security_utils.h +147 -0
- data/src/core/tsi/ssl_types.h +27 -27
- data/src/core/tsi/transport_security.cc +44 -32
- data/src/core/tsi/transport_security.h +49 -48
- data/src/core/tsi/transport_security_grpc.cc +23 -22
- data/src/core/tsi/transport_security_grpc.h +44 -41
- data/src/core/tsi/transport_security_interface.h +346 -332
- data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.clang +2 -0
- data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.gcc +7 -0
- data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.clang +2 -0
- data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.gcc +7 -0
- data/src/ruby/ext/grpc/ext-export.gcc +1 -1
- data/src/ruby/ext/grpc/extconf.rb +98 -20
- data/src/ruby/ext/grpc/rb_call.c +1 -0
- data/src/ruby/ext/grpc/rb_channel.c +1 -0
- data/src/ruby/ext/grpc/rb_channel_args.c +1 -0
- data/src/ruby/ext/grpc/rb_compression_options.c +1 -1
- data/src/ruby/ext/grpc/rb_grpc.c +1 -0
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.c +38 -38
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +60 -60
- data/src/ruby/ext/grpc/rb_loader.c +6 -2
- data/src/ruby/lib/grpc/errors.rb +1 -1
- data/src/ruby/lib/grpc/generic/active_call.rb +7 -1
- data/src/ruby/lib/grpc/generic/bidi_call.rb +2 -0
- data/src/ruby/lib/grpc/grpc.rb +1 -1
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/src/ruby/pb/generate_proto_ruby.sh +1 -6
- data/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +4 -0
- data/src/ruby/pb/test/client.rb +769 -0
- data/src/ruby/pb/test/server.rb +252 -0
- data/src/ruby/pb/test/xds_client.rb +415 -0
- data/src/ruby/spec/channel_spec.rb +5 -43
- data/src/ruby/spec/client_server_spec.rb +20 -8
- data/src/ruby/spec/generic/active_call_spec.rb +12 -3
- data/src/ruby/spec/generic/client_stub_spec.rb +23 -23
- data/src/ruby/spec/generic/server_interceptors_spec.rb +1 -1
- data/src/ruby/spec/user_agent_spec.rb +1 -1
- data/third_party/abseil-cpp/absl/algorithm/container.h +57 -58
- data/third_party/abseil-cpp/absl/base/attributes.h +88 -41
- data/third_party/abseil-cpp/absl/base/casts.h +61 -68
- data/third_party/abseil-cpp/absl/base/config.h +221 -39
- data/third_party/abseil-cpp/absl/base/internal/cycleclock.cc +12 -42
- data/third_party/abseil-cpp/absl/base/internal/cycleclock.h +52 -2
- data/third_party/abseil-cpp/absl/base/internal/cycleclock_config.h +55 -0
- data/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +5 -4
- data/third_party/abseil-cpp/absl/base/internal/endian.h +17 -62
- data/third_party/abseil-cpp/absl/base/internal/fast_type_id.h +2 -0
- data/third_party/abseil-cpp/absl/base/internal/inline_variable.h +3 -3
- data/third_party/abseil-cpp/absl/base/internal/invoke.h +54 -0
- data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +2 -2
- data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.h +1 -1
- data/third_party/abseil-cpp/absl/base/internal/prefetch.h +138 -0
- data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc +39 -28
- data/third_party/abseil-cpp/absl/base/internal/raw_logging.h +36 -36
- data/third_party/abseil-cpp/absl/base/internal/spinlock.cc +6 -3
- data/third_party/abseil-cpp/absl/base/internal/spinlock.h +10 -6
- data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc +2 -5
- data/third_party/abseil-cpp/absl/base/internal/spinlock_win32.inc +4 -1
- data/third_party/abseil-cpp/absl/base/internal/strerror.cc +88 -0
- data/third_party/abseil-cpp/absl/base/internal/strerror.h +39 -0
- data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc +14 -11
- data/third_party/abseil-cpp/absl/base/internal/thread_annotations.h +9 -0
- data/third_party/abseil-cpp/absl/base/internal/thread_identity.cc +2 -1
- data/third_party/abseil-cpp/absl/base/internal/thread_identity.h +4 -0
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc +6 -7
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.h +12 -40
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock_config.h +62 -0
- data/third_party/abseil-cpp/absl/base/log_severity.cc +28 -0
- data/third_party/abseil-cpp/absl/base/log_severity.h +51 -0
- data/third_party/abseil-cpp/absl/base/macros.h +4 -21
- data/third_party/abseil-cpp/absl/base/optimization.h +76 -16
- data/third_party/abseil-cpp/absl/base/options.h +1 -7
- data/third_party/abseil-cpp/absl/base/policy_checks.h +15 -13
- data/third_party/abseil-cpp/absl/base/thread_annotations.h +2 -2
- data/third_party/abseil-cpp/absl/cleanup/cleanup.h +140 -0
- data/third_party/abseil-cpp/absl/cleanup/internal/cleanup.h +100 -0
- data/third_party/abseil-cpp/absl/container/fixed_array.h +9 -5
- data/third_party/abseil-cpp/absl/container/flat_hash_map.h +11 -4
- data/third_party/abseil-cpp/absl/container/flat_hash_set.h +503 -0
- data/third_party/abseil-cpp/absl/container/inlined_vector.h +84 -25
- data/third_party/abseil-cpp/absl/container/internal/common.h +9 -8
- data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h +132 -0
- data/third_party/abseil-cpp/absl/container/internal/container_memory.h +23 -29
- data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h +4 -55
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc +116 -23
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h +40 -54
- data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h +165 -66
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc +159 -4
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h +1121 -470
- data/third_party/abseil-cpp/absl/crc/crc32c.cc +99 -0
- data/third_party/abseil-cpp/absl/crc/crc32c.h +183 -0
- data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc +256 -0
- data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.h +57 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc.cc +468 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc.h +91 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +269 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32c.h +39 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc32c_inline.h +72 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc +130 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.h +159 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_internal.h +179 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy.h +119 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +75 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc +434 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc +93 -0
- data/third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +725 -0
- data/third_party/abseil-cpp/absl/crc/internal/non_temporal_arm_intrinsics.h +79 -0
- data/third_party/abseil-cpp/absl/crc/internal/non_temporal_memcpy.h +180 -0
- data/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc +45 -88
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.cc +67 -38
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.h +1 -1
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc +16 -13
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h +3 -2
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +19 -12
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_arm-inl.inc +8 -3
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_config.h +2 -1
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_generic-inl.inc +1 -1
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc +8 -3
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_riscv-inl.inc +59 -102
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_win32-inl.inc +5 -4
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_x86-inl.inc +41 -11
- data/third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc +18 -4
- data/third_party/abseil-cpp/absl/debugging/symbolize.cc +6 -1
- data/third_party/abseil-cpp/absl/debugging/symbolize_darwin.inc +3 -2
- data/third_party/abseil-cpp/absl/debugging/symbolize_elf.inc +164 -101
- data/third_party/abseil-cpp/absl/debugging/symbolize_win32.inc +7 -6
- data/third_party/abseil-cpp/absl/functional/any_invocable.h +316 -0
- data/third_party/abseil-cpp/absl/functional/bind_front.h +10 -1
- data/third_party/abseil-cpp/absl/functional/function_ref.h +2 -1
- data/third_party/abseil-cpp/absl/functional/internal/any_invocable.h +878 -0
- data/third_party/abseil-cpp/absl/hash/hash.h +82 -8
- data/third_party/abseil-cpp/absl/hash/internal/city.cc +10 -10
- data/third_party/abseil-cpp/absl/hash/internal/hash.h +235 -26
- data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc +3 -14
- data/third_party/abseil-cpp/absl/memory/memory.h +26 -447
- data/third_party/abseil-cpp/absl/meta/type_traits.h +104 -12
- data/third_party/abseil-cpp/absl/numeric/bits.h +1 -1
- data/third_party/abseil-cpp/absl/numeric/int128.cc +14 -10
- data/third_party/abseil-cpp/absl/numeric/int128.h +2 -2
- data/third_party/abseil-cpp/absl/profiling/internal/sample_recorder.h +33 -10
- data/third_party/abseil-cpp/absl/random/bernoulli_distribution.h +4 -4
- data/third_party/abseil-cpp/absl/random/distributions.h +3 -3
- data/third_party/abseil-cpp/absl/random/internal/distribution_caller.h +3 -0
- data/third_party/abseil-cpp/absl/random/internal/fast_uniform_bits.h +4 -2
- data/third_party/abseil-cpp/absl/random/internal/generate_real.h +2 -2
- data/third_party/abseil-cpp/absl/random/internal/nonsecure_base.h +59 -48
- data/third_party/abseil-cpp/absl/random/internal/pcg_engine.h +3 -24
- data/third_party/abseil-cpp/absl/random/internal/pool_urbg.cc +9 -9
- data/third_party/abseil-cpp/absl/random/internal/randen.h +5 -11
- data/third_party/abseil-cpp/absl/random/internal/randen_detect.cc +6 -2
- data/third_party/abseil-cpp/absl/random/internal/randen_engine.h +48 -23
- data/third_party/abseil-cpp/absl/random/internal/salted_seed_seq.h +24 -26
- data/third_party/abseil-cpp/absl/random/internal/seed_material.cc +2 -2
- data/third_party/abseil-cpp/absl/random/internal/traits.h +53 -5
- data/third_party/abseil-cpp/absl/random/internal/uniform_helper.h +5 -5
- data/third_party/abseil-cpp/absl/random/internal/wide_multiply.h +33 -48
- data/third_party/abseil-cpp/absl/random/log_uniform_int_distribution.h +9 -10
- data/third_party/abseil-cpp/absl/random/poisson_distribution.h +7 -4
- data/third_party/abseil-cpp/absl/random/random.h +6 -6
- data/third_party/abseil-cpp/absl/random/seed_sequences.h +1 -0
- data/third_party/abseil-cpp/absl/random/uniform_int_distribution.h +2 -2
- data/third_party/abseil-cpp/absl/random/uniform_real_distribution.h +1 -1
- data/third_party/abseil-cpp/absl/random/zipf_distribution.h +4 -3
- data/third_party/abseil-cpp/absl/status/internal/status_internal.h +18 -0
- data/third_party/abseil-cpp/absl/status/status.cc +193 -14
- data/third_party/abseil-cpp/absl/status/status.h +24 -14
- data/third_party/abseil-cpp/absl/status/statusor.h +9 -3
- data/third_party/abseil-cpp/absl/strings/ascii.cc +5 -5
- data/third_party/abseil-cpp/absl/strings/ascii.h +4 -4
- data/third_party/abseil-cpp/absl/strings/charconv.cc +534 -96
- data/third_party/abseil-cpp/absl/strings/cord.cc +257 -924
- data/third_party/abseil-cpp/absl/strings/cord.h +268 -156
- data/third_party/abseil-cpp/absl/strings/cord_analysis.cc +188 -0
- data/third_party/abseil-cpp/absl/strings/cord_analysis.h +44 -0
- data/third_party/abseil-cpp/absl/strings/cord_buffer.cc +30 -0
- data/third_party/abseil-cpp/absl/strings/cord_buffer.h +575 -0
- data/third_party/abseil-cpp/absl/strings/escaping.cc +73 -62
- data/third_party/abseil-cpp/absl/strings/escaping.h +24 -19
- data/third_party/abseil-cpp/absl/strings/internal/char_map.h +14 -12
- data/third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc +4 -4
- data/third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc +2 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_data_edge.h +63 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.cc +20 -32
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +441 -150
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc +157 -53
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h +70 -73
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc +8 -6
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.h +4 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc +3 -2
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.h +5 -4
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc +7 -74
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc +56 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.h +103 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_flat.h +58 -17
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_ring.cc +13 -11
- data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.h +7 -15
- data/third_party/abseil-cpp/absl/strings/internal/cordz_handle.h +3 -3
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.cc +19 -43
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.h +7 -7
- data/third_party/abseil-cpp/absl/strings/internal/cordz_statistics.h +5 -4
- data/third_party/abseil-cpp/absl/strings/internal/cordz_update_tracker.h +4 -2
- data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc +93 -0
- data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.h +34 -0
- data/third_party/abseil-cpp/absl/strings/internal/escaping.cc +18 -15
- data/third_party/abseil-cpp/absl/strings/internal/escaping.h +7 -9
- data/third_party/abseil-cpp/absl/strings/internal/has_absl_stringify.h +55 -0
- data/third_party/abseil-cpp/absl/strings/internal/memutil.cc +9 -6
- data/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +14 -7
- data/third_party/abseil-cpp/absl/strings/internal/ostringstream.h +35 -10
- data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc +113 -46
- data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.h +127 -30
- data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc +3 -2
- data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.h +42 -10
- data/third_party/abseil-cpp/absl/strings/internal/str_format/checker.h +56 -289
- data/third_party/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h +351 -0
- data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc +6 -6
- data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.h +9 -4
- data/third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc +215 -181
- data/third_party/abseil-cpp/absl/strings/internal/str_format/output.h +2 -1
- data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc +10 -209
- data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.h +14 -103
- data/third_party/abseil-cpp/absl/strings/internal/str_join_internal.h +9 -6
- data/third_party/abseil-cpp/absl/strings/internal/str_split_internal.h +2 -1
- data/third_party/abseil-cpp/absl/strings/internal/string_constant.h +10 -2
- data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc +28 -0
- data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.h +57 -0
- data/third_party/abseil-cpp/absl/strings/internal/utf8.cc +9 -9
- data/third_party/abseil-cpp/absl/strings/numbers.cc +42 -39
- data/third_party/abseil-cpp/absl/strings/numbers.h +26 -23
- data/third_party/abseil-cpp/absl/strings/str_cat.cc +9 -6
- data/third_party/abseil-cpp/absl/strings/str_cat.h +70 -16
- data/third_party/abseil-cpp/absl/strings/str_format.h +71 -9
- data/third_party/abseil-cpp/absl/strings/str_join.h +9 -15
- data/third_party/abseil-cpp/absl/strings/str_split.h +1 -2
- data/third_party/abseil-cpp/absl/strings/string_view.cc +8 -19
- data/third_party/abseil-cpp/absl/strings/string_view.h +6 -12
- data/third_party/abseil-cpp/absl/strings/strip.h +8 -6
- data/third_party/abseil-cpp/absl/strings/substitute.cc +8 -6
- data/third_party/abseil-cpp/absl/strings/substitute.h +55 -21
- data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc +9 -6
- data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.h +0 -4
- data/third_party/abseil-cpp/absl/synchronization/internal/futex.h +20 -17
- data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc +37 -31
- data/third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.h +22 -8
- data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc +0 -4
- data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.h +1 -6
- data/third_party/abseil-cpp/absl/synchronization/internal/waiter.cc +0 -25
- data/third_party/abseil-cpp/absl/synchronization/internal/waiter.h +10 -4
- data/third_party/abseil-cpp/absl/synchronization/mutex.cc +172 -88
- data/third_party/abseil-cpp/absl/synchronization/mutex.h +102 -55
- data/third_party/abseil-cpp/absl/synchronization/notification.cc +0 -1
- data/third_party/abseil-cpp/absl/synchronization/notification.h +3 -3
- data/third_party/abseil-cpp/absl/time/civil_time.cc +26 -0
- data/third_party/abseil-cpp/absl/time/civil_time.h +25 -0
- data/third_party/abseil-cpp/absl/time/clock.cc +17 -11
- data/third_party/abseil-cpp/absl/time/duration.cc +12 -11
- data/third_party/abseil-cpp/absl/time/format.cc +2 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h +11 -7
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc +1 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc +26 -5
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc +7 -6
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc +36 -35
- data/third_party/abseil-cpp/absl/time/time.cc +2 -2
- data/third_party/abseil-cpp/absl/time/time.h +268 -169
- data/third_party/abseil-cpp/absl/types/internal/optional.h +8 -0
- data/third_party/abseil-cpp/absl/types/internal/span.h +30 -19
- data/third_party/abseil-cpp/absl/types/internal/variant.h +28 -40
- data/third_party/abseil-cpp/absl/types/optional.h +17 -14
- data/third_party/abseil-cpp/absl/types/span.h +31 -8
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_genn.c +1 -1
- data/third_party/boringssl-with-bazel/src/include/openssl/x509v3.h +1 -2
- data/third_party/re2/re2/bitstate.cc +3 -3
- data/third_party/re2/re2/dfa.cc +13 -13
- data/third_party/re2/re2/nfa.cc +4 -4
- data/third_party/re2/re2/onepass.cc +2 -2
- data/third_party/re2/re2/prefilter_tree.cc +27 -59
- data/third_party/re2/re2/prefilter_tree.h +3 -2
- data/third_party/re2/re2/prog.cc +11 -2
- data/third_party/re2/re2/prog.h +17 -5
- data/third_party/re2/re2/re2.cc +6 -11
- data/third_party/re2/re2/re2.h +1 -1
- data/third_party/re2/re2/regexp.cc +1 -2
- data/third_party/re2/re2/stringpiece.h +10 -7
- data/third_party/re2/re2/unicode_casefold.cc +25 -11
- data/third_party/re2/re2/unicode_groups.cc +319 -151
- data/third_party/re2/re2/walker-inl.h +3 -2
- data/third_party/re2/util/mutex.h +4 -4
- data/third_party/upb/upb/arena.c +277 -0
- data/third_party/upb/upb/arena.h +225 -0
- data/third_party/upb/upb/array.c +114 -0
- data/third_party/upb/upb/array.h +83 -0
- data/third_party/upb/upb/collections.h +36 -0
- data/third_party/upb/upb/decode.c +188 -76
- data/third_party/upb/upb/decode.h +1 -0
- data/third_party/upb/upb/decode_fast.c +1 -1
- data/third_party/upb/upb/def.c +128 -57
- data/third_party/upb/upb/def.h +20 -4
- data/third_party/upb/upb/def.hpp +7 -4
- data/third_party/upb/upb/encode.c +43 -28
- data/third_party/upb/upb/encode.h +16 -6
- data/third_party/upb/upb/extension_registry.c +93 -0
- data/third_party/upb/upb/extension_registry.h +84 -0
- data/third_party/upb/upb/{decode_internal.h → internal/decode.h} +5 -5
- data/third_party/upb/upb/internal/table.h +385 -0
- data/third_party/upb/upb/{upb_internal.h → internal/upb.h} +3 -3
- data/third_party/upb/upb/internal/vsnprintf_compat.h +52 -0
- data/third_party/upb/upb/json_decode.c +1512 -0
- data/third_party/upb/upb/json_decode.h +47 -0
- data/third_party/upb/upb/json_encode.c +780 -0
- data/third_party/upb/upb/json_encode.h +65 -0
- data/third_party/upb/upb/map.c +108 -0
- data/third_party/upb/upb/map.h +117 -0
- data/third_party/upb/upb/message_value.h +66 -0
- data/third_party/upb/upb/mini_table.c +1147 -0
- data/third_party/upb/upb/mini_table.h +189 -0
- data/third_party/upb/upb/mini_table.hpp +112 -0
- data/third_party/upb/upb/msg.c +6 -68
- data/third_party/upb/upb/msg.h +2 -46
- data/third_party/upb/upb/msg_internal.h +77 -58
- data/third_party/upb/upb/port_def.inc +10 -1
- data/third_party/upb/upb/port_undef.inc +2 -0
- data/third_party/upb/upb/reflection.c +2 -159
- data/third_party/upb/upb/reflection.h +2 -112
- data/third_party/upb/upb/status.c +86 -0
- data/third_party/upb/upb/status.h +66 -0
- data/third_party/upb/upb/table.c +12 -8
- data/third_party/upb/upb/table_internal.h +3 -350
- data/third_party/upb/upb/text_encode.c +3 -2
- data/third_party/upb/upb/upb.c +4 -290
- data/third_party/upb/upb/upb.h +8 -167
- data/third_party/zlib/compress.c +3 -3
- data/third_party/zlib/crc32.c +975 -292
- data/third_party/zlib/crc32.h +9441 -436
- data/third_party/zlib/deflate.c +183 -129
- data/third_party/zlib/deflate.h +12 -15
- data/third_party/zlib/gzguts.h +3 -2
- data/third_party/zlib/gzlib.c +6 -4
- data/third_party/zlib/gzread.c +8 -12
- data/third_party/zlib/gzwrite.c +26 -14
- data/third_party/zlib/infback.c +12 -8
- data/third_party/zlib/inffast.c +14 -14
- data/third_party/zlib/inflate.c +44 -10
- data/third_party/zlib/inflate.h +3 -2
- data/third_party/zlib/inftrees.c +3 -3
- data/third_party/zlib/inftrees.h +1 -1
- data/third_party/zlib/trees.c +85 -107
- data/third_party/zlib/uncompr.c +2 -2
- data/third_party/zlib/zconf.h +16 -3
- data/third_party/zlib/zlib.h +129 -106
- data/third_party/zlib/zutil.c +11 -9
- data/third_party/zlib/zutil.h +13 -9
- metadata +424 -135
- data/include/grpc/impl/codegen/gpr_slice.h +0 -71
- data/src/core/ext/filters/client_channel/http_connect_handshaker.h +0 -42
- data/src/core/ext/filters/client_channel/lb_policy_factory.h +0 -48
- data/src/core/ext/filters/client_channel/lb_policy_registry.cc +0 -185
- data/src/core/ext/filters/client_channel/lb_policy_registry.h +0 -65
- data/src/core/ext/filters/client_channel/proxy_mapper.h +0 -54
- data/src/core/ext/filters/client_channel/proxy_mapper_registry.cc +0 -89
- data/src/core/ext/filters/client_channel/proxy_mapper_registry.h +0 -50
- data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +0 -186
- data/src/core/ext/filters/client_idle/client_idle_filter.cc +0 -201
- data/src/core/ext/filters/fault_injection/service_config_parser.cc +0 -179
- data/src/core/ext/filters/http/message_compress/message_compress_filter.cc +0 -456
- data/src/core/ext/filters/http/message_compress/message_compress_filter.h +0 -53
- data/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +0 -386
- data/src/core/ext/filters/max_age/max_age_filter.cc +0 -566
- data/src/core/ext/filters/server_config_selector/server_config_selector.cc +0 -67
- data/src/core/ext/xds/certificate_provider_registry.cc +0 -103
- data/src/core/ext/xds/certificate_provider_registry.h +0 -57
- data/src/core/lib/channel/handshaker_factory.h +0 -50
- data/src/core/lib/channel/handshaker_registry.cc +0 -50
- data/src/core/lib/channel/handshaker_registry.h +0 -71
- data/src/core/lib/event_engine/sockaddr.cc +0 -40
- data/src/core/lib/event_engine/sockaddr.h +0 -44
- data/src/core/lib/gpr/env.h +0 -40
- data/src/core/lib/gpr/env_linux.cc +0 -75
- data/src/core/lib/gpr/env_posix.cc +0 -46
- data/src/core/lib/gpr/env_windows.cc +0 -74
- data/src/core/lib/gpr/murmur_hash.cc +0 -82
- data/src/core/lib/gpr/murmur_hash.h +0 -29
- data/src/core/lib/gpr/string_windows.h +0 -32
- data/src/core/lib/gpr/tls.h +0 -157
- data/src/core/lib/gprpp/capture.h +0 -76
- data/src/core/lib/iomgr/endpoint_pair_event_engine.cc +0 -32
- data/src/core/lib/iomgr/error_internal.h +0 -66
- data/src/core/lib/iomgr/ev_epollex_linux.cc +0 -1657
- data/src/core/lib/iomgr/ev_epollex_linux.h +0 -30
- data/src/core/lib/iomgr/event_engine/endpoint.cc +0 -172
- data/src/core/lib/iomgr/event_engine/endpoint.h +0 -52
- data/src/core/lib/iomgr/event_engine/iomgr.cc +0 -85
- data/src/core/lib/iomgr/event_engine/pollset.cc +0 -87
- data/src/core/lib/iomgr/event_engine/resolved_address_internal.cc +0 -47
- data/src/core/lib/iomgr/event_engine/resolver.cc +0 -133
- data/src/core/lib/iomgr/event_engine/resolver.h +0 -56
- data/src/core/lib/iomgr/event_engine/tcp.cc +0 -296
- data/src/core/lib/iomgr/event_engine/timer.cc +0 -62
- data/src/core/lib/iomgr/executor/mpmcqueue.cc +0 -182
- data/src/core/lib/iomgr/executor/mpmcqueue.h +0 -171
- data/src/core/lib/iomgr/executor/threadpool.cc +0 -136
- data/src/core/lib/iomgr/executor/threadpool.h +0 -150
- data/src/core/lib/iomgr/is_epollexclusive_available.cc +0 -119
- data/src/core/lib/iomgr/is_epollexclusive_available.h +0 -36
- data/src/core/lib/iomgr/sys_epoll_wrapper.h +0 -30
- data/src/core/lib/iomgr/time_averaged_stats.cc +0 -64
- data/src/core/lib/iomgr/time_averaged_stats.h +0 -72
- data/src/core/lib/profiling/basic_timers.cc +0 -295
- data/src/core/lib/profiling/stap_timers.cc +0 -50
- data/src/core/lib/profiling/timers.h +0 -94
- data/src/core/lib/security/security_connector/load_system_roots_linux.h +0 -46
- data/src/core/lib/slice/slice_api.cc +0 -39
- data/src/core/lib/slice/slice_refcount_base.h +0 -61
- data/src/core/lib/slice/slice_split.cc +0 -100
- data/src/core/lib/slice/slice_split.h +0 -40
- data/src/core/lib/transport/byte_stream.cc +0 -162
- data/src/core/lib/transport/byte_stream.h +0 -166
- data/third_party/abseil-cpp/absl/container/internal/have_sse.h +0 -50
@@ -1,58 +1,95 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
1
|
+
//
|
2
|
+
//
|
3
|
+
// Copyright 2015 gRPC authors.
|
4
|
+
//
|
5
|
+
// Licensed under the Apache License, Version 2.0 (the "License");
|
6
|
+
// you may not use this file except in compliance with the License.
|
7
|
+
// You may obtain a copy of the License at
|
8
|
+
//
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
//
|
11
|
+
// Unless required by applicable law or agreed to in writing, software
|
12
|
+
// distributed under the License is distributed on an "AS IS" BASIS,
|
13
|
+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
14
|
+
// See the License for the specific language governing permissions and
|
15
|
+
// limitations under the License.
|
16
|
+
//
|
17
|
+
//
|
18
18
|
|
19
19
|
#include <grpc/support/port_platform.h>
|
20
20
|
|
21
21
|
#include "src/core/lib/surface/call.h"
|
22
22
|
|
23
|
-
#include <
|
23
|
+
#include <inttypes.h>
|
24
24
|
#include <limits.h>
|
25
|
-
#include <stdio.h>
|
26
25
|
#include <stdlib.h>
|
27
|
-
#include <string.h>
|
28
26
|
|
27
|
+
#include <algorithm>
|
28
|
+
#include <atomic>
|
29
|
+
#include <initializer_list>
|
30
|
+
#include <memory>
|
31
|
+
#include <new>
|
29
32
|
#include <string>
|
30
|
-
|
33
|
+
#include <type_traits>
|
34
|
+
#include <utility>
|
35
|
+
#include <vector>
|
36
|
+
|
37
|
+
#include "absl/base/thread_annotations.h"
|
38
|
+
#include "absl/cleanup/cleanup.h"
|
39
|
+
#include "absl/meta/type_traits.h"
|
40
|
+
#include "absl/status/status.h"
|
31
41
|
#include "absl/strings/str_cat.h"
|
32
42
|
#include "absl/strings/str_format.h"
|
43
|
+
#include "absl/strings/str_join.h"
|
44
|
+
#include "absl/strings/string_view.h"
|
45
|
+
#include "absl/types/variant.h"
|
33
46
|
|
47
|
+
#include <grpc/byte_buffer.h>
|
34
48
|
#include <grpc/compression.h>
|
49
|
+
#include <grpc/event_engine/event_engine.h>
|
35
50
|
#include <grpc/grpc.h>
|
51
|
+
#include <grpc/impl/propagation_bits.h>
|
36
52
|
#include <grpc/slice.h>
|
53
|
+
#include <grpc/slice_buffer.h>
|
54
|
+
#include <grpc/status.h>
|
37
55
|
#include <grpc/support/alloc.h>
|
56
|
+
#include <grpc/support/atm.h>
|
38
57
|
#include <grpc/support/log.h>
|
39
58
|
#include <grpc/support/string_util.h>
|
59
|
+
#include <grpc/support/time.h>
|
40
60
|
|
61
|
+
#include "src/core/lib/channel/call_finalization.h"
|
41
62
|
#include "src/core/lib/channel/channel_stack.h"
|
63
|
+
#include "src/core/lib/channel/channelz.h"
|
64
|
+
#include "src/core/lib/channel/context.h"
|
65
|
+
#include "src/core/lib/channel/status_util.h"
|
42
66
|
#include "src/core/lib/compression/compression_internal.h"
|
43
67
|
#include "src/core/lib/debug/stats.h"
|
68
|
+
#include "src/core/lib/debug/stats_data.h"
|
69
|
+
#include "src/core/lib/experiments/experiments.h"
|
44
70
|
#include "src/core/lib/gpr/alloc.h"
|
45
|
-
#include "src/core/lib/gpr/string.h"
|
46
71
|
#include "src/core/lib/gpr/time_precise.h"
|
47
72
|
#include "src/core/lib/gpr/useful.h"
|
48
|
-
#include "src/core/lib/gprpp/
|
73
|
+
#include "src/core/lib/gprpp/bitset.h"
|
74
|
+
#include "src/core/lib/gprpp/cpp_impl_of.h"
|
75
|
+
#include "src/core/lib/gprpp/crash.h"
|
76
|
+
#include "src/core/lib/gprpp/debug_location.h"
|
49
77
|
#include "src/core/lib/gprpp/ref_counted.h"
|
50
|
-
#include "src/core/lib/
|
51
|
-
#include "src/core/lib/
|
78
|
+
#include "src/core/lib/gprpp/ref_counted_ptr.h"
|
79
|
+
#include "src/core/lib/gprpp/status_helper.h"
|
80
|
+
#include "src/core/lib/gprpp/sync.h"
|
81
|
+
#include "src/core/lib/iomgr/call_combiner.h"
|
82
|
+
#include "src/core/lib/iomgr/exec_ctx.h"
|
83
|
+
#include "src/core/lib/iomgr/polling_entity.h"
|
84
|
+
#include "src/core/lib/promise/activity.h"
|
85
|
+
#include "src/core/lib/promise/arena_promise.h"
|
86
|
+
#include "src/core/lib/promise/context.h"
|
87
|
+
#include "src/core/lib/promise/detail/basic_seq.h"
|
88
|
+
#include "src/core/lib/promise/pipe.h"
|
89
|
+
#include "src/core/lib/promise/poll.h"
|
52
90
|
#include "src/core/lib/resource_quota/arena.h"
|
91
|
+
#include "src/core/lib/slice/slice_buffer.h"
|
53
92
|
#include "src/core/lib/slice/slice_internal.h"
|
54
|
-
#include "src/core/lib/slice/slice_split.h"
|
55
|
-
#include "src/core/lib/slice/slice_string_helpers.h"
|
56
93
|
#include "src/core/lib/surface/api_trace.h"
|
57
94
|
#include "src/core/lib/surface/call_test_only.h"
|
58
95
|
#include "src/core/lib/surface/channel.h"
|
@@ -60,172 +97,531 @@
|
|
60
97
|
#include "src/core/lib/surface/server.h"
|
61
98
|
#include "src/core/lib/surface/validate_metadata.h"
|
62
99
|
#include "src/core/lib/transport/error_utils.h"
|
100
|
+
#include "src/core/lib/transport/metadata_batch.h"
|
63
101
|
#include "src/core/lib/transport/transport.h"
|
64
102
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
103
|
+
grpc_core::TraceFlag grpc_call_error_trace(false, "call_error");
|
104
|
+
grpc_core::TraceFlag grpc_compression_trace(false, "compression");
|
105
|
+
grpc_core::TraceFlag grpc_call_trace(false, "call");
|
106
|
+
grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount");
|
107
|
+
|
108
|
+
namespace grpc_core {
|
109
|
+
|
110
|
+
///////////////////////////////////////////////////////////////////////////////
|
111
|
+
// Call
|
112
|
+
|
113
|
+
class Call : public CppImplOf<Call, grpc_call> {
|
114
|
+
public:
|
115
|
+
Arena* arena() { return arena_; }
|
116
|
+
bool is_client() const { return is_client_; }
|
117
|
+
|
118
|
+
virtual void ContextSet(grpc_context_index elem, void* value,
|
119
|
+
void (*destroy)(void* value)) = 0;
|
120
|
+
virtual void* ContextGet(grpc_context_index elem) const = 0;
|
121
|
+
virtual bool Completed() = 0;
|
122
|
+
void CancelWithStatus(grpc_status_code status, const char* description);
|
123
|
+
virtual void CancelWithError(grpc_error_handle error) = 0;
|
124
|
+
virtual void SetCompletionQueue(grpc_completion_queue* cq) = 0;
|
125
|
+
char* GetPeer();
|
126
|
+
virtual grpc_call_error StartBatch(const grpc_op* ops, size_t nops,
|
127
|
+
void* notify_tag,
|
128
|
+
bool is_notify_tag_closure) = 0;
|
129
|
+
virtual bool failed_before_recv_message() const = 0;
|
130
|
+
virtual bool is_trailers_only() const = 0;
|
131
|
+
virtual absl::string_view GetServerAuthority() const = 0;
|
132
|
+
virtual void ExternalRef() = 0;
|
133
|
+
virtual void ExternalUnref() = 0;
|
134
|
+
virtual void InternalRef(const char* reason) = 0;
|
135
|
+
virtual void InternalUnref(const char* reason) = 0;
|
136
|
+
|
137
|
+
virtual grpc_compression_algorithm test_only_compression_algorithm() = 0;
|
138
|
+
virtual uint32_t test_only_message_flags() = 0;
|
139
|
+
virtual uint32_t test_only_encodings_accepted_by_peer() = 0;
|
140
|
+
virtual grpc_compression_algorithm compression_for_level(
|
141
|
+
grpc_compression_level level) = 0;
|
142
|
+
|
143
|
+
// This should return nullptr for the promise stack (and alternative means
|
144
|
+
// for that functionality be invented)
|
145
|
+
virtual grpc_call_stack* call_stack() = 0;
|
146
|
+
|
147
|
+
gpr_atm* peer_string_atm_ptr() { return &peer_string_; }
|
148
|
+
|
149
|
+
protected:
|
150
|
+
// The maximum number of concurrent batches possible.
|
151
|
+
// Based upon the maximum number of individually queueable ops in the batch
|
152
|
+
// api:
|
153
|
+
// - initial metadata send
|
154
|
+
// - message send
|
155
|
+
// - status/close send (depending on client/server)
|
156
|
+
// - initial metadata recv
|
157
|
+
// - message recv
|
158
|
+
// - status/close recv (depending on client/server)
|
159
|
+
static constexpr size_t kMaxConcurrentBatches = 6;
|
160
|
+
|
161
|
+
struct ParentCall {
|
162
|
+
Mutex child_list_mu;
|
163
|
+
Call* first_child ABSL_GUARDED_BY(child_list_mu) = nullptr;
|
164
|
+
};
|
165
|
+
|
166
|
+
struct ChildCall {
|
167
|
+
explicit ChildCall(Call* parent) : parent(parent) {}
|
168
|
+
Call* parent;
|
169
|
+
/// siblings: children of the same parent form a list, and this list is
|
170
|
+
/// protected under
|
171
|
+
/// parent->mu
|
172
|
+
Call* sibling_next = nullptr;
|
173
|
+
Call* sibling_prev = nullptr;
|
174
|
+
};
|
175
|
+
|
176
|
+
Call(Arena* arena, bool is_client, Timestamp send_deadline,
|
177
|
+
RefCountedPtr<Channel> channel)
|
178
|
+
: channel_(std::move(channel)),
|
179
|
+
arena_(arena),
|
180
|
+
send_deadline_(send_deadline),
|
181
|
+
is_client_(is_client) {
|
182
|
+
GPR_DEBUG_ASSERT(arena_ != nullptr);
|
183
|
+
GPR_DEBUG_ASSERT(channel_ != nullptr);
|
184
|
+
}
|
185
|
+
virtual ~Call() = default;
|
186
|
+
|
187
|
+
void DeleteThis();
|
188
|
+
|
189
|
+
ParentCall* GetOrCreateParentCall();
|
190
|
+
ParentCall* parent_call();
|
191
|
+
Channel* channel() {
|
192
|
+
GPR_DEBUG_ASSERT(channel_ != nullptr);
|
193
|
+
return channel_.get();
|
194
|
+
}
|
195
|
+
|
196
|
+
absl::Status InitParent(Call* parent, uint32_t propagation_mask);
|
197
|
+
void PublishToParent(Call* parent);
|
198
|
+
void MaybeUnpublishFromParent();
|
199
|
+
void PropagateCancellationToChildren();
|
200
|
+
|
201
|
+
Timestamp send_deadline() const { return send_deadline_; }
|
202
|
+
void set_send_deadline(Timestamp send_deadline) {
|
203
|
+
send_deadline_ = send_deadline;
|
204
|
+
}
|
205
|
+
|
206
|
+
void ClearPeerString() { gpr_atm_rel_store(&peer_string_, 0); }
|
207
|
+
|
208
|
+
private:
|
209
|
+
RefCountedPtr<Channel> channel_;
|
210
|
+
Arena* const arena_;
|
211
|
+
std::atomic<ParentCall*> parent_call_{nullptr};
|
212
|
+
ChildCall* child_ = nullptr;
|
213
|
+
Timestamp send_deadline_;
|
214
|
+
const bool is_client_;
|
215
|
+
// flag indicating that cancellation is inherited
|
216
|
+
bool cancellation_is_inherited_ = false;
|
217
|
+
// A char* indicating the peer name.
|
218
|
+
gpr_atm peer_string_ = 0;
|
219
|
+
};
|
220
|
+
|
221
|
+
Call::ParentCall* Call::GetOrCreateParentCall() {
|
222
|
+
ParentCall* p = parent_call_.load(std::memory_order_acquire);
|
223
|
+
if (p == nullptr) {
|
224
|
+
p = arena_->New<ParentCall>();
|
225
|
+
ParentCall* expected = nullptr;
|
226
|
+
if (!parent_call_.compare_exchange_strong(expected, p,
|
227
|
+
std::memory_order_release,
|
228
|
+
std::memory_order_relaxed)) {
|
229
|
+
p->~ParentCall();
|
230
|
+
p = expected;
|
231
|
+
}
|
232
|
+
}
|
233
|
+
return p;
|
234
|
+
}
|
235
|
+
|
236
|
+
Call::ParentCall* Call::parent_call() {
|
237
|
+
return parent_call_.load(std::memory_order_acquire);
|
238
|
+
}
|
239
|
+
|
240
|
+
absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
|
241
|
+
child_ = arena()->New<ChildCall>(parent);
|
242
|
+
|
243
|
+
parent->InternalRef("child");
|
244
|
+
GPR_ASSERT(is_client_);
|
245
|
+
GPR_ASSERT(!parent->is_client_);
|
246
|
+
|
247
|
+
if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
|
248
|
+
send_deadline_ = std::min(send_deadline_, parent->send_deadline_);
|
249
|
+
}
|
250
|
+
// for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
|
251
|
+
// GRPC_PROPAGATE_STATS_CONTEXT
|
252
|
+
// TODO(ctiller): This should change to use the appropriate census start_op
|
253
|
+
// call.
|
254
|
+
if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
|
255
|
+
if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
|
256
|
+
return absl::UnknownError(
|
257
|
+
"Census tracing propagation requested without Census context "
|
258
|
+
"propagation");
|
259
|
+
}
|
260
|
+
ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
|
261
|
+
nullptr);
|
262
|
+
} else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
263
|
+
return absl::UnknownError(
|
264
|
+
"Census context propagation requested without Census tracing "
|
265
|
+
"propagation");
|
106
266
|
}
|
107
|
-
|
108
|
-
|
267
|
+
if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
|
268
|
+
cancellation_is_inherited_ = true;
|
109
269
|
}
|
110
|
-
|
270
|
+
return absl::OkStatus();
|
271
|
+
}
|
272
|
+
|
273
|
+
void Call::PublishToParent(Call* parent) {
|
274
|
+
ChildCall* cc = child_;
|
275
|
+
ParentCall* pc = parent->GetOrCreateParentCall();
|
276
|
+
MutexLock lock(&pc->child_list_mu);
|
277
|
+
if (pc->first_child == nullptr) {
|
278
|
+
pc->first_child = this;
|
279
|
+
cc->sibling_next = cc->sibling_prev = this;
|
280
|
+
} else {
|
281
|
+
cc->sibling_next = pc->first_child;
|
282
|
+
cc->sibling_prev = pc->first_child->child_->sibling_prev;
|
283
|
+
cc->sibling_next->child_->sibling_prev =
|
284
|
+
cc->sibling_prev->child_->sibling_next = this;
|
285
|
+
}
|
286
|
+
if (parent->Completed()) {
|
287
|
+
CancelWithError(absl::CancelledError());
|
288
|
+
}
|
289
|
+
}
|
111
290
|
|
112
|
-
|
113
|
-
|
114
|
-
|
291
|
+
void Call::MaybeUnpublishFromParent() {
|
292
|
+
ChildCall* cc = child_;
|
293
|
+
if (cc == nullptr) return;
|
115
294
|
|
116
|
-
|
117
|
-
|
118
|
-
|
295
|
+
ParentCall* pc = cc->parent->parent_call();
|
296
|
+
{
|
297
|
+
MutexLock lock(&pc->child_list_mu);
|
298
|
+
if (this == pc->first_child) {
|
299
|
+
pc->first_child = cc->sibling_next;
|
300
|
+
if (this == pc->first_child) {
|
301
|
+
pc->first_child = nullptr;
|
302
|
+
}
|
303
|
+
}
|
304
|
+
cc->sibling_prev->child_->sibling_next = cc->sibling_next;
|
305
|
+
cc->sibling_next->child_->sibling_prev = cc->sibling_prev;
|
306
|
+
}
|
307
|
+
cc->parent->InternalUnref("child");
|
308
|
+
}
|
119
309
|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
310
|
+
void Call::CancelWithStatus(grpc_status_code status, const char* description) {
|
311
|
+
// copying 'description' is needed to ensure the grpc_call_cancel_with_status
|
312
|
+
// guarantee that can be short-lived.
|
313
|
+
CancelWithError(grpc_error_set_int(
|
314
|
+
grpc_error_set_str(GRPC_ERROR_CREATE(description),
|
315
|
+
StatusStrProperty::kGrpcMessage, description),
|
316
|
+
StatusIntProperty::kRpcStatus, status));
|
317
|
+
}
|
318
|
+
|
319
|
+
void Call::PropagateCancellationToChildren() {
|
320
|
+
ParentCall* pc = parent_call();
|
321
|
+
if (pc != nullptr) {
|
322
|
+
Call* child;
|
323
|
+
MutexLock lock(&pc->child_list_mu);
|
324
|
+
child = pc->first_child;
|
325
|
+
if (child != nullptr) {
|
326
|
+
do {
|
327
|
+
Call* next_child_call = child->child_->sibling_next;
|
328
|
+
if (child->cancellation_is_inherited_) {
|
329
|
+
child->InternalRef("propagate_cancel");
|
330
|
+
child->CancelWithError(absl::CancelledError());
|
331
|
+
child->InternalUnref("propagate_cancel");
|
332
|
+
}
|
333
|
+
child = next_child_call;
|
334
|
+
} while (child != pc->first_child);
|
335
|
+
}
|
336
|
+
}
|
337
|
+
}
|
338
|
+
|
339
|
+
char* Call::GetPeer() {
|
340
|
+
char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
|
341
|
+
if (peer_string != nullptr) return gpr_strdup(peer_string);
|
342
|
+
peer_string = grpc_channel_get_target(channel_->c_ptr());
|
343
|
+
if (peer_string != nullptr) return peer_string;
|
344
|
+
return gpr_strdup("unknown");
|
345
|
+
}
|
129
346
|
|
130
|
-
|
131
|
-
|
347
|
+
void Call::DeleteThis() {
|
348
|
+
RefCountedPtr<Channel> channel = std::move(channel_);
|
349
|
+
Arena* arena = arena_;
|
350
|
+
this->~Call();
|
351
|
+
channel->UpdateCallSizeEstimate(arena->TotalUsedBytes());
|
352
|
+
arena->Destroy();
|
353
|
+
}
|
132
354
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
cq(args.cq),
|
137
|
-
channel(args.channel),
|
138
|
-
is_client(args.server_transport_data == nullptr),
|
139
|
-
stream_op_payload(context) {}
|
355
|
+
///////////////////////////////////////////////////////////////////////////////
|
356
|
+
// FilterStackCall
|
357
|
+
// To be removed once promise conversion is complete
|
140
358
|
|
141
|
-
|
359
|
+
class FilterStackCall final : public Call {
|
360
|
+
public:
|
361
|
+
~FilterStackCall() override {
|
142
362
|
for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
|
143
|
-
if (
|
144
|
-
|
363
|
+
if (context_[i].destroy) {
|
364
|
+
context_[i].destroy(context_[i].value);
|
145
365
|
}
|
146
366
|
}
|
147
|
-
gpr_free(static_cast<void*>(const_cast<char*>(
|
148
|
-
}
|
149
|
-
|
150
|
-
grpc_core::RefCount ext_ref;
|
151
|
-
grpc_core::Arena* arena;
|
152
|
-
grpc_core::CallCombiner call_combiner;
|
153
|
-
grpc_completion_queue* cq;
|
154
|
-
grpc_polling_entity pollent;
|
155
|
-
grpc_channel* channel;
|
156
|
-
gpr_cycle_counter start_time = gpr_get_cycle_counter();
|
157
|
-
/* parent_call* */ gpr_atm parent_call_atm = 0;
|
158
|
-
child_call* child = nullptr;
|
159
|
-
|
160
|
-
/* client or server call */
|
161
|
-
bool is_client;
|
162
|
-
/** has grpc_call_unref been called */
|
163
|
-
bool destroy_called = false;
|
164
|
-
/** flag indicating that cancellation is inherited */
|
165
|
-
bool cancellation_is_inherited = false;
|
166
|
-
// Trailers-only response status
|
167
|
-
bool is_trailers_only = false;
|
168
|
-
/** which ops are in-flight */
|
169
|
-
bool sent_initial_metadata = false;
|
170
|
-
bool sending_message = false;
|
171
|
-
bool sent_final_op = false;
|
172
|
-
bool received_initial_metadata = false;
|
173
|
-
bool receiving_message = false;
|
174
|
-
bool requested_final_op = false;
|
175
|
-
gpr_atm any_ops_sent_atm = 0;
|
176
|
-
gpr_atm received_final_op_atm = 0;
|
177
|
-
|
178
|
-
batch_control* active_batches[MAX_CONCURRENT_BATCHES] = {};
|
179
|
-
grpc_transport_stream_op_batch_payload stream_op_payload;
|
180
|
-
|
181
|
-
/* first idx: is_receiving, second idx: is_trailing */
|
182
|
-
grpc_metadata_batch send_initial_metadata{arena};
|
183
|
-
grpc_metadata_batch send_trailing_metadata{arena};
|
184
|
-
grpc_metadata_batch recv_initial_metadata{arena};
|
185
|
-
grpc_metadata_batch recv_trailing_metadata{arena};
|
186
|
-
|
187
|
-
/* Buffered read metadata waiting to be returned to the application.
|
188
|
-
Element 0 is initial metadata, element 1 is trailing metadata. */
|
189
|
-
grpc_metadata_array* buffered_metadata[2] = {};
|
367
|
+
gpr_free(static_cast<void*>(const_cast<char*>(final_info_.error_string)));
|
368
|
+
}
|
190
369
|
|
191
|
-
|
192
|
-
|
370
|
+
bool Completed() override {
|
371
|
+
return gpr_atm_acq_load(&received_final_op_atm_) != 0;
|
372
|
+
}
|
373
|
+
|
374
|
+
// TODO(ctiller): return absl::StatusOr<SomeSmartPointer<Call>>?
|
375
|
+
static grpc_error_handle Create(grpc_call_create_args* args,
|
376
|
+
grpc_call** out_call);
|
377
|
+
|
378
|
+
static Call* FromTopElem(grpc_call_element* elem) {
|
379
|
+
return FromCallStack(grpc_call_stack_from_top_element(elem));
|
380
|
+
}
|
381
|
+
|
382
|
+
grpc_call_stack* call_stack() override {
|
383
|
+
return reinterpret_cast<grpc_call_stack*>(
|
384
|
+
reinterpret_cast<char*>(this) +
|
385
|
+
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(*this)));
|
386
|
+
}
|
387
|
+
|
388
|
+
grpc_call_element* call_elem(size_t idx) {
|
389
|
+
return grpc_call_stack_element(call_stack(), idx);
|
390
|
+
}
|
391
|
+
|
392
|
+
CallCombiner* call_combiner() { return &call_combiner_; }
|
393
|
+
|
394
|
+
void CancelWithError(grpc_error_handle error) override;
|
395
|
+
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
396
|
+
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
397
|
+
bool is_notify_tag_closure) override;
|
398
|
+
void ExternalRef() override { ext_ref_.Ref(); }
|
399
|
+
void ExternalUnref() override;
|
400
|
+
void InternalRef(const char* reason) override {
|
401
|
+
GRPC_CALL_STACK_REF(call_stack(), reason);
|
402
|
+
}
|
403
|
+
void InternalUnref(const char* reason) override {
|
404
|
+
GRPC_CALL_STACK_UNREF(call_stack(), reason);
|
405
|
+
}
|
406
|
+
|
407
|
+
void ContextSet(grpc_context_index elem, void* value,
|
408
|
+
void (*destroy)(void* value)) override;
|
409
|
+
void* ContextGet(grpc_context_index elem) const override {
|
410
|
+
return context_[elem].value;
|
411
|
+
}
|
412
|
+
|
413
|
+
grpc_compression_algorithm compression_for_level(
|
414
|
+
grpc_compression_level level) override {
|
415
|
+
return encodings_accepted_by_peer_.CompressionAlgorithmForLevel(level);
|
416
|
+
}
|
417
|
+
|
418
|
+
bool is_trailers_only() const override {
|
419
|
+
bool result = is_trailers_only_;
|
420
|
+
GPR_DEBUG_ASSERT(!result || recv_initial_metadata_.TransportSize() == 0);
|
421
|
+
return result;
|
422
|
+
}
|
423
|
+
|
424
|
+
bool failed_before_recv_message() const override {
|
425
|
+
return call_failed_before_recv_message_;
|
426
|
+
}
|
427
|
+
|
428
|
+
absl::string_view GetServerAuthority() const override {
|
429
|
+
const Slice* authority_metadata =
|
430
|
+
recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
|
431
|
+
if (authority_metadata == nullptr) return "";
|
432
|
+
return authority_metadata->as_string_view();
|
433
|
+
}
|
434
|
+
|
435
|
+
grpc_compression_algorithm test_only_compression_algorithm() override {
|
436
|
+
return incoming_compression_algorithm_;
|
437
|
+
}
|
438
|
+
|
439
|
+
uint32_t test_only_message_flags() override {
|
440
|
+
return test_only_last_message_flags_;
|
441
|
+
}
|
442
|
+
|
443
|
+
uint32_t test_only_encodings_accepted_by_peer() override {
|
444
|
+
return encodings_accepted_by_peer_.ToLegacyBitmask();
|
445
|
+
}
|
446
|
+
|
447
|
+
static size_t InitialSizeEstimate() {
|
448
|
+
return sizeof(FilterStackCall) +
|
449
|
+
sizeof(BatchControl) * kMaxConcurrentBatches;
|
450
|
+
}
|
193
451
|
|
194
|
-
|
195
|
-
|
196
|
-
|
452
|
+
private:
|
453
|
+
static constexpr gpr_atm kRecvNone = 0;
|
454
|
+
static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
|
455
|
+
|
456
|
+
enum class PendingOp {
|
457
|
+
kRecvMessage,
|
458
|
+
kRecvInitialMetadata,
|
459
|
+
kRecvTrailingMetadata,
|
460
|
+
kSends
|
461
|
+
};
|
462
|
+
static intptr_t PendingOpMask(PendingOp op) {
|
463
|
+
return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
|
464
|
+
}
|
465
|
+
static std::string PendingOpString(intptr_t pending_ops) {
|
466
|
+
std::vector<absl::string_view> pending_op_strings;
|
467
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
|
468
|
+
pending_op_strings.push_back("kRecvMessage");
|
469
|
+
}
|
470
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
|
471
|
+
pending_op_strings.push_back("kRecvInitialMetadata");
|
472
|
+
}
|
473
|
+
if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
|
474
|
+
pending_op_strings.push_back("kRecvTrailingMetadata");
|
475
|
+
}
|
476
|
+
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
477
|
+
pending_op_strings.push_back("kSends");
|
478
|
+
}
|
479
|
+
return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
|
480
|
+
}
|
481
|
+
struct BatchControl {
|
482
|
+
FilterStackCall* call_ = nullptr;
|
483
|
+
grpc_transport_stream_op_batch op_;
|
484
|
+
// Share memory for cq_completion and notify_tag as they are never needed
|
485
|
+
// simultaneously. Each byte used in this data structure count as six bytes
|
486
|
+
// per call, so any savings we can make are worthwhile,
|
487
|
+
|
488
|
+
// We use notify_tag to determine whether or not to send notification to the
|
489
|
+
// completion queue. Once we've made that determination, we can reuse the
|
490
|
+
// memory for cq_completion.
|
491
|
+
union {
|
492
|
+
grpc_cq_completion cq_completion;
|
493
|
+
struct {
|
494
|
+
// Any given op indicates completion by either (a) calling a closure or
|
495
|
+
// (b) sending a notification on the call's completion queue. If
|
496
|
+
// \a is_closure is true, \a tag indicates a closure to be invoked;
|
497
|
+
// otherwise, \a tag indicates the tag to be used in the notification to
|
498
|
+
// be sent to the completion queue.
|
499
|
+
void* tag;
|
500
|
+
bool is_closure;
|
501
|
+
} notify_tag;
|
502
|
+
} completion_data_;
|
503
|
+
grpc_closure start_batch_;
|
504
|
+
grpc_closure finish_batch_;
|
505
|
+
std::atomic<intptr_t> ops_pending_{0};
|
506
|
+
AtomicError batch_error_;
|
507
|
+
void set_pending_ops(uintptr_t ops) {
|
508
|
+
ops_pending_.store(ops, std::memory_order_release);
|
509
|
+
}
|
510
|
+
bool completed_batch_step(PendingOp op) {
|
511
|
+
auto mask = PendingOpMask(op);
|
512
|
+
auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
|
513
|
+
if (grpc_call_trace.enabled()) {
|
514
|
+
gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
|
515
|
+
PendingOpString(mask).c_str(),
|
516
|
+
PendingOpString(r & ~mask).c_str(),
|
517
|
+
completion_data_.notify_tag.tag);
|
518
|
+
}
|
519
|
+
GPR_ASSERT((r & mask) != 0);
|
520
|
+
return r == mask;
|
521
|
+
}
|
197
522
|
|
198
|
-
|
199
|
-
|
523
|
+
void PostCompletion();
|
524
|
+
void FinishStep(PendingOp op);
|
525
|
+
void ProcessDataAfterMetadata();
|
526
|
+
void ReceivingStreamReady(grpc_error_handle error);
|
527
|
+
void ValidateFilteredMetadata();
|
528
|
+
void ReceivingInitialMetadataReady(grpc_error_handle error);
|
529
|
+
void ReceivingTrailingMetadataReady(grpc_error_handle error);
|
530
|
+
void FinishBatch(grpc_error_handle error);
|
531
|
+
};
|
532
|
+
|
533
|
+
FilterStackCall(Arena* arena, const grpc_call_create_args& args)
|
534
|
+
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
535
|
+
args.channel->Ref()),
|
536
|
+
cq_(args.cq),
|
537
|
+
stream_op_payload_(context_) {}
|
538
|
+
|
539
|
+
static void ReleaseCall(void* call, grpc_error_handle);
|
540
|
+
static void DestroyCall(void* call, grpc_error_handle);
|
541
|
+
|
542
|
+
static FilterStackCall* FromCallStack(grpc_call_stack* call_stack) {
|
543
|
+
return reinterpret_cast<FilterStackCall*>(
|
544
|
+
reinterpret_cast<char*>(call_stack) -
|
545
|
+
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)));
|
546
|
+
}
|
547
|
+
|
548
|
+
void ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
549
|
+
grpc_closure* start_batch_closure);
|
550
|
+
void SetFinalStatus(grpc_error_handle error);
|
551
|
+
BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
|
552
|
+
void HandleCompressionAlgorithmDisabled(
|
553
|
+
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
554
|
+
void HandleCompressionAlgorithmNotAccepted(
|
555
|
+
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
556
|
+
bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
|
557
|
+
bool is_trailing);
|
558
|
+
void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
|
559
|
+
void RecvInitialFilter(grpc_metadata_batch* b);
|
560
|
+
void RecvTrailingFilter(grpc_metadata_batch* b,
|
561
|
+
grpc_error_handle batch_error);
|
562
|
+
|
563
|
+
RefCount ext_ref_;
|
564
|
+
CallCombiner call_combiner_;
|
565
|
+
grpc_completion_queue* cq_;
|
566
|
+
grpc_polling_entity pollent_;
|
567
|
+
gpr_cycle_counter start_time_ = gpr_get_cycle_counter();
|
568
|
+
|
569
|
+
/// has grpc_call_unref been called
|
570
|
+
bool destroy_called_ = false;
|
571
|
+
// Trailers-only response status
|
572
|
+
bool is_trailers_only_ = false;
|
573
|
+
/// which ops are in-flight
|
574
|
+
bool sent_initial_metadata_ = false;
|
575
|
+
bool sending_message_ = false;
|
576
|
+
bool sent_final_op_ = false;
|
577
|
+
bool received_initial_metadata_ = false;
|
578
|
+
bool receiving_message_ = false;
|
579
|
+
bool requested_final_op_ = false;
|
580
|
+
gpr_atm received_final_op_atm_ = 0;
|
581
|
+
|
582
|
+
BatchControl* active_batches_[kMaxConcurrentBatches] = {};
|
583
|
+
grpc_transport_stream_op_batch_payload stream_op_payload_;
|
584
|
+
|
585
|
+
// first idx: is_receiving, second idx: is_trailing
|
586
|
+
grpc_metadata_batch send_initial_metadata_{arena()};
|
587
|
+
grpc_metadata_batch send_trailing_metadata_{arena()};
|
588
|
+
grpc_metadata_batch recv_initial_metadata_{arena()};
|
589
|
+
grpc_metadata_batch recv_trailing_metadata_{arena()};
|
590
|
+
|
591
|
+
// Buffered read metadata waiting to be returned to the application.
|
592
|
+
// Element 0 is initial metadata, element 1 is trailing metadata.
|
593
|
+
grpc_metadata_array* buffered_metadata_[2] = {};
|
594
|
+
|
595
|
+
// Call data useful used for reporting. Only valid after the call has
|
596
|
+
// completed
|
597
|
+
grpc_call_final_info final_info_;
|
598
|
+
|
599
|
+
// Compression algorithm for *incoming* data
|
600
|
+
grpc_compression_algorithm incoming_compression_algorithm_ =
|
200
601
|
GRPC_COMPRESS_NONE;
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
grpc_closure receiving_slice_ready;
|
220
|
-
grpc_closure receiving_stream_ready;
|
221
|
-
grpc_closure receiving_initial_metadata_ready;
|
222
|
-
grpc_closure receiving_trailing_metadata_ready;
|
223
|
-
uint32_t test_only_last_message_flags = 0;
|
602
|
+
// Supported encodings (compression algorithms), a bitset.
|
603
|
+
// Always support no compression.
|
604
|
+
CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
|
605
|
+
|
606
|
+
// Contexts for various subsystems (security, tracing, ...).
|
607
|
+
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
608
|
+
|
609
|
+
SliceBuffer send_slice_buffer_;
|
610
|
+
absl::optional<SliceBuffer> receiving_slice_buffer_;
|
611
|
+
uint32_t receiving_stream_flags_;
|
612
|
+
|
613
|
+
bool call_failed_before_recv_message_ = false;
|
614
|
+
grpc_byte_buffer** receiving_buffer_ = nullptr;
|
615
|
+
grpc_slice receiving_slice_ = grpc_empty_slice();
|
616
|
+
grpc_closure receiving_stream_ready_;
|
617
|
+
grpc_closure receiving_initial_metadata_ready_;
|
618
|
+
grpc_closure receiving_trailing_metadata_ready_;
|
619
|
+
uint32_t test_only_last_message_flags_ = 0;
|
224
620
|
// Status about operation of call
|
225
|
-
bool
|
226
|
-
gpr_atm
|
621
|
+
bool sent_server_trailing_metadata_ = false;
|
622
|
+
gpr_atm cancelled_with_error_ = 0;
|
227
623
|
|
228
|
-
grpc_closure
|
624
|
+
grpc_closure release_call_;
|
229
625
|
|
230
626
|
union {
|
231
627
|
struct {
|
@@ -236,572 +632,332 @@ struct grpc_call {
|
|
236
632
|
struct {
|
237
633
|
int* cancelled;
|
238
634
|
// backpointer to owning server if this is a server side call.
|
239
|
-
|
635
|
+
Server* core_server;
|
240
636
|
} server;
|
241
|
-
}
|
242
|
-
AtomicError
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
gpr_atm
|
637
|
+
} final_op_;
|
638
|
+
AtomicError status_error_;
|
639
|
+
|
640
|
+
// recv_state can contain one of the following values:
|
641
|
+
// RECV_NONE : : no initial metadata and messages received
|
642
|
+
// RECV_INITIAL_METADATA_FIRST : received initial metadata first
|
643
|
+
// a batch_control* : received messages first
|
644
|
+
|
645
|
+
// +------1------RECV_NONE------3-----+
|
646
|
+
// | |
|
647
|
+
// | |
|
648
|
+
// v v
|
649
|
+
// RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
|
650
|
+
// | ^ | ^
|
651
|
+
// | | | |
|
652
|
+
// +-----2-----+ +-----4-----+
|
653
|
+
|
654
|
+
// For 1, 4: See receiving_initial_metadata_ready() function
|
655
|
+
// For 2, 3: See receiving_stream_ready() function
|
656
|
+
gpr_atm recv_state_ = 0;
|
261
657
|
};
|
262
658
|
|
263
|
-
|
264
|
-
|
659
|
+
grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
660
|
+
grpc_call** out_call) {
|
661
|
+
Channel* channel = args->channel.get();
|
265
662
|
|
266
|
-
|
267
|
-
(grpc_call_stack*)((char*)(call) + \
|
268
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call)))
|
269
|
-
#define CALL_FROM_CALL_STACK(call_stack) \
|
270
|
-
(grpc_call*)(((char*)(call_stack)) - \
|
271
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call)))
|
272
|
-
|
273
|
-
#define CALL_ELEM_FROM_CALL(call, idx) \
|
274
|
-
grpc_call_stack_element(CALL_STACK_FROM_CALL(call), idx)
|
275
|
-
#define CALL_FROM_TOP_ELEM(top_elem) \
|
276
|
-
CALL_FROM_CALL_STACK(grpc_call_stack_from_top_element(top_elem))
|
277
|
-
|
278
|
-
static void execute_batch(grpc_call* call,
|
279
|
-
grpc_transport_stream_op_batch* batch,
|
280
|
-
grpc_closure* start_batch_closure);
|
281
|
-
|
282
|
-
static void cancel_with_status(grpc_call* c, grpc_status_code status,
|
283
|
-
const char* description);
|
284
|
-
static void cancel_with_error(grpc_call* c, grpc_error_handle error);
|
285
|
-
static void destroy_call(void* call_stack, grpc_error_handle error);
|
286
|
-
static void receiving_slice_ready(void* bctlp, grpc_error_handle error);
|
287
|
-
static void set_final_status(grpc_call* call, grpc_error_handle error);
|
288
|
-
static void process_data_after_md(batch_control* bctl);
|
289
|
-
static void post_batch_completion(batch_control* bctl);
|
290
|
-
|
291
|
-
static void add_init_error(grpc_error_handle* composite,
|
663
|
+
auto add_init_error = [](grpc_error_handle* composite,
|
292
664
|
grpc_error_handle new_err) {
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
}
|
297
|
-
*composite = grpc_error_add_child(*composite, new_err);
|
298
|
-
}
|
299
|
-
|
300
|
-
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
301
|
-
grpc_core::ExecCtx exec_ctx;
|
302
|
-
return call->arena->Alloc(size);
|
303
|
-
}
|
304
|
-
|
305
|
-
static parent_call* get_or_create_parent_call(grpc_call* call) {
|
306
|
-
parent_call* p =
|
307
|
-
reinterpret_cast<parent_call*>(gpr_atm_acq_load(&call->parent_call_atm));
|
308
|
-
if (p == nullptr) {
|
309
|
-
p = call->arena->New<parent_call>();
|
310
|
-
if (!gpr_atm_rel_cas(&call->parent_call_atm,
|
311
|
-
reinterpret_cast<gpr_atm>(nullptr),
|
312
|
-
reinterpret_cast<gpr_atm>(p))) {
|
313
|
-
p->~parent_call();
|
314
|
-
p = reinterpret_cast<parent_call*>(
|
315
|
-
gpr_atm_acq_load(&call->parent_call_atm));
|
665
|
+
if (new_err.ok()) return;
|
666
|
+
if (composite->ok()) {
|
667
|
+
*composite = GRPC_ERROR_CREATE("Call creation failed");
|
316
668
|
}
|
317
|
-
|
318
|
-
|
319
|
-
}
|
320
|
-
|
321
|
-
static parent_call* get_parent_call(grpc_call* call) {
|
322
|
-
return reinterpret_cast<parent_call*>(
|
323
|
-
gpr_atm_acq_load(&call->parent_call_atm));
|
324
|
-
}
|
325
|
-
|
326
|
-
size_t grpc_call_get_initial_size_estimate() {
|
327
|
-
return sizeof(grpc_call) + sizeof(batch_control) * MAX_CONCURRENT_BATCHES;
|
328
|
-
}
|
669
|
+
*composite = grpc_error_add_child(*composite, new_err);
|
670
|
+
};
|
329
671
|
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
grpc_core::Arena* arena;
|
337
|
-
grpc_call* call;
|
338
|
-
grpc_error_handle error = GRPC_ERROR_NONE;
|
339
|
-
grpc_channel_stack* channel_stack =
|
340
|
-
grpc_channel_get_channel_stack(args->channel);
|
341
|
-
size_t initial_size = grpc_channel_get_call_size_estimate(args->channel);
|
342
|
-
GRPC_STATS_INC_CALL_INITIAL_SIZE(initial_size);
|
343
|
-
size_t call_and_stack_size =
|
344
|
-
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call)) +
|
345
|
-
channel_stack->call_stack_size;
|
672
|
+
Arena* arena;
|
673
|
+
FilterStackCall* call;
|
674
|
+
grpc_error_handle error;
|
675
|
+
grpc_channel_stack* channel_stack = channel->channel_stack();
|
676
|
+
size_t initial_size = channel->CallSizeEstimate();
|
677
|
+
global_stats().IncrementCallInitialSize(initial_size);
|
346
678
|
size_t call_alloc_size =
|
347
|
-
|
679
|
+
GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
|
680
|
+
channel_stack->call_stack_size;
|
348
681
|
|
349
|
-
std::pair<
|
350
|
-
|
351
|
-
&*args->channel->allocator);
|
682
|
+
std::pair<Arena*, void*> arena_with_call = Arena::CreateWithAlloc(
|
683
|
+
initial_size, call_alloc_size, channel->allocator());
|
352
684
|
arena = arena_with_call.first;
|
353
|
-
call = new (arena_with_call.second)
|
354
|
-
|
685
|
+
call = new (arena_with_call.second) FilterStackCall(arena, *args);
|
686
|
+
GPR_DEBUG_ASSERT(FromC(call->c_ptr()) == call);
|
687
|
+
GPR_DEBUG_ASSERT(FromCallStack(call->call_stack()) == call);
|
688
|
+
*out_call = call->c_ptr();
|
355
689
|
grpc_slice path = grpc_empty_slice();
|
356
|
-
if (call->is_client) {
|
357
|
-
call->
|
358
|
-
call->
|
359
|
-
call->
|
360
|
-
|
361
|
-
path =
|
362
|
-
call->
|
363
|
-
|
690
|
+
if (call->is_client()) {
|
691
|
+
call->final_op_.client.status_details = nullptr;
|
692
|
+
call->final_op_.client.status = nullptr;
|
693
|
+
call->final_op_.client.error_string = nullptr;
|
694
|
+
global_stats().IncrementClientCallsCreated();
|
695
|
+
path = CSliceRef(args->path->c_slice());
|
696
|
+
call->send_initial_metadata_.Set(HttpPathMetadata(),
|
697
|
+
std::move(*args->path));
|
364
698
|
if (args->authority.has_value()) {
|
365
|
-
call->
|
366
|
-
|
699
|
+
call->send_initial_metadata_.Set(HttpAuthorityMetadata(),
|
700
|
+
std::move(*args->authority));
|
367
701
|
}
|
368
702
|
} else {
|
369
|
-
|
370
|
-
call->
|
371
|
-
call->
|
372
|
-
}
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
send_deadline = std::min(send_deadline, args->parent->send_deadline);
|
387
|
-
}
|
388
|
-
/* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
|
389
|
-
* GRPC_PROPAGATE_STATS_CONTEXT */
|
390
|
-
/* TODO(ctiller): This should change to use the appropriate census start_op
|
391
|
-
* call. */
|
392
|
-
if (args->propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
|
393
|
-
if (0 == (args->propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
|
394
|
-
add_init_error(&error, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
|
395
|
-
"Census tracing propagation requested "
|
396
|
-
"without Census context propagation"));
|
397
|
-
}
|
398
|
-
grpc_call_context_set(call, GRPC_CONTEXT_TRACING,
|
399
|
-
args->parent->context[GRPC_CONTEXT_TRACING].value,
|
400
|
-
nullptr);
|
401
|
-
} else if (args->propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
|
402
|
-
add_init_error(&error, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
|
403
|
-
"Census context propagation requested "
|
404
|
-
"without Census tracing propagation"));
|
405
|
-
}
|
406
|
-
if (args->propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
|
407
|
-
call->cancellation_is_inherited = true;
|
408
|
-
if (gpr_atm_acq_load(&args->parent->received_final_op_atm)) {
|
409
|
-
immediately_cancel = true;
|
410
|
-
}
|
411
|
-
}
|
412
|
-
}
|
413
|
-
call->send_deadline = send_deadline;
|
414
|
-
/* initial refcount dropped by grpc_call_unref */
|
415
|
-
grpc_call_element_args call_args = {CALL_STACK_FROM_CALL(call),
|
416
|
-
args->server_transport_data,
|
417
|
-
call->context,
|
418
|
-
path,
|
419
|
-
call->start_time,
|
420
|
-
send_deadline,
|
421
|
-
call->arena,
|
422
|
-
&call->call_combiner};
|
423
|
-
add_init_error(&error, grpc_call_stack_init(channel_stack, 1, destroy_call,
|
703
|
+
global_stats().IncrementServerCallsCreated();
|
704
|
+
call->final_op_.server.cancelled = nullptr;
|
705
|
+
call->final_op_.server.core_server = args->server;
|
706
|
+
}
|
707
|
+
|
708
|
+
Call* parent = Call::FromC(args->parent);
|
709
|
+
if (parent != nullptr) {
|
710
|
+
add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
|
711
|
+
parent, args->propagation_mask)));
|
712
|
+
}
|
713
|
+
// initial refcount dropped by grpc_call_unref
|
714
|
+
grpc_call_element_args call_args = {
|
715
|
+
call->call_stack(), args->server_transport_data,
|
716
|
+
call->context_, path,
|
717
|
+
call->start_time_, call->send_deadline(),
|
718
|
+
call->arena(), &call->call_combiner_};
|
719
|
+
add_init_error(&error, grpc_call_stack_init(channel_stack, 1, DestroyCall,
|
424
720
|
call, &call_args));
|
425
721
|
// Publish this call to parent only after the call stack has been initialized.
|
426
|
-
if (
|
427
|
-
|
428
|
-
parent_call* pc = get_or_create_parent_call(args->parent);
|
429
|
-
gpr_mu_lock(&pc->child_list_mu);
|
430
|
-
if (pc->first_child == nullptr) {
|
431
|
-
pc->first_child = call;
|
432
|
-
cc->sibling_next = cc->sibling_prev = call;
|
433
|
-
} else {
|
434
|
-
cc->sibling_next = pc->first_child;
|
435
|
-
cc->sibling_prev = pc->first_child->child->sibling_prev;
|
436
|
-
cc->sibling_next->child->sibling_prev =
|
437
|
-
cc->sibling_prev->child->sibling_next = call;
|
438
|
-
}
|
439
|
-
gpr_mu_unlock(&pc->child_list_mu);
|
722
|
+
if (parent != nullptr) {
|
723
|
+
call->PublishToParent(parent);
|
440
724
|
}
|
441
725
|
|
442
|
-
if (error
|
443
|
-
|
444
|
-
}
|
445
|
-
if (immediately_cancel) {
|
446
|
-
cancel_with_error(call, GRPC_ERROR_CANCELLED);
|
726
|
+
if (!error.ok()) {
|
727
|
+
call->CancelWithError(error);
|
447
728
|
}
|
448
729
|
if (args->cq != nullptr) {
|
449
730
|
GPR_ASSERT(args->pollset_set_alternative == nullptr &&
|
450
731
|
"Only one of 'cq' and 'pollset_set_alternative' should be "
|
451
732
|
"non-nullptr.");
|
452
733
|
GRPC_CQ_INTERNAL_REF(args->cq, "bind");
|
453
|
-
call->
|
734
|
+
call->pollent_ =
|
454
735
|
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args->cq));
|
455
736
|
}
|
456
737
|
if (args->pollset_set_alternative != nullptr) {
|
457
|
-
call->
|
738
|
+
call->pollent_ = grpc_polling_entity_create_from_pollset_set(
|
458
739
|
args->pollset_set_alternative);
|
459
740
|
}
|
460
|
-
if (!grpc_polling_entity_is_empty(&call->
|
461
|
-
grpc_call_stack_set_pollset_or_pollset_set(
|
462
|
-
&call->
|
741
|
+
if (!grpc_polling_entity_is_empty(&call->pollent_)) {
|
742
|
+
grpc_call_stack_set_pollset_or_pollset_set(call->call_stack(),
|
743
|
+
&call->pollent_);
|
463
744
|
}
|
464
745
|
|
465
|
-
if (call->is_client) {
|
466
|
-
|
467
|
-
grpc_channel_get_channelz_node(call->channel);
|
746
|
+
if (call->is_client()) {
|
747
|
+
channelz::ChannelNode* channelz_channel = channel->channelz_node();
|
468
748
|
if (channelz_channel != nullptr) {
|
469
749
|
channelz_channel->RecordCallStarted();
|
470
750
|
}
|
471
|
-
} else if (call->
|
472
|
-
|
473
|
-
call->
|
751
|
+
} else if (call->final_op_.server.core_server != nullptr) {
|
752
|
+
channelz::ServerNode* channelz_node =
|
753
|
+
call->final_op_.server.core_server->channelz_node();
|
474
754
|
if (channelz_node != nullptr) {
|
475
755
|
channelz_node->RecordCallStarted();
|
476
756
|
}
|
477
757
|
}
|
478
758
|
|
479
|
-
|
759
|
+
CSliceUnref(path);
|
480
760
|
|
481
761
|
return error;
|
482
762
|
}
|
483
763
|
|
484
|
-
void
|
485
|
-
grpc_completion_queue* cq) {
|
764
|
+
void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
486
765
|
GPR_ASSERT(cq);
|
487
766
|
|
488
|
-
if (grpc_polling_entity_pollset_set(&
|
489
|
-
|
490
|
-
abort();
|
767
|
+
if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
|
768
|
+
Crash("A pollset_set is already registered for this call.");
|
491
769
|
}
|
492
|
-
|
770
|
+
cq_ = cq;
|
493
771
|
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
494
|
-
|
495
|
-
grpc_call_stack_set_pollset_or_pollset_set(
|
496
|
-
&call->pollent);
|
497
|
-
}
|
498
|
-
|
499
|
-
#ifndef NDEBUG
|
500
|
-
#define REF_REASON reason
|
501
|
-
#define REF_ARG , const char* reason
|
502
|
-
#else
|
503
|
-
#define REF_REASON ""
|
504
|
-
#define REF_ARG
|
505
|
-
#endif
|
506
|
-
void grpc_call_internal_ref(grpc_call* c REF_ARG) {
|
507
|
-
GRPC_CALL_STACK_REF(CALL_STACK_FROM_CALL(c), REF_REASON);
|
508
|
-
}
|
509
|
-
void grpc_call_internal_unref(grpc_call* c REF_ARG) {
|
510
|
-
GRPC_CALL_STACK_UNREF(CALL_STACK_FROM_CALL(c), REF_REASON);
|
772
|
+
pollent_ = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
|
773
|
+
grpc_call_stack_set_pollset_or_pollset_set(call_stack(), &pollent_);
|
511
774
|
}
|
512
775
|
|
513
|
-
|
514
|
-
|
515
|
-
grpc_channel* channel = c->channel;
|
516
|
-
grpc_core::Arena* arena = c->arena;
|
517
|
-
c->~grpc_call();
|
518
|
-
grpc_channel_update_call_size_estimate(channel, arena->Destroy());
|
519
|
-
GRPC_CHANNEL_INTERNAL_UNREF(channel, "call");
|
776
|
+
void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
|
777
|
+
static_cast<FilterStackCall*>(call)->DeleteThis();
|
520
778
|
}
|
521
779
|
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
c->
|
526
|
-
c->
|
527
|
-
c->
|
528
|
-
parent_call* pc = get_parent_call(c);
|
780
|
+
void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
|
781
|
+
auto* c = static_cast<FilterStackCall*>(call);
|
782
|
+
c->recv_initial_metadata_.Clear();
|
783
|
+
c->recv_trailing_metadata_.Clear();
|
784
|
+
c->receiving_slice_buffer_.reset();
|
785
|
+
ParentCall* pc = c->parent_call();
|
529
786
|
if (pc != nullptr) {
|
530
|
-
pc->~
|
787
|
+
pc->~ParentCall();
|
531
788
|
}
|
532
|
-
if (c->
|
533
|
-
GRPC_CQ_INTERNAL_UNREF(c->
|
789
|
+
if (c->cq_) {
|
790
|
+
GRPC_CQ_INTERNAL_UNREF(c->cq_, "bind");
|
534
791
|
}
|
535
792
|
|
536
|
-
grpc_error_handle status_error = c->
|
537
|
-
grpc_error_get_status(status_error, c->send_deadline,
|
538
|
-
&c->
|
539
|
-
&(c->
|
540
|
-
c->
|
541
|
-
c->
|
542
|
-
gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->
|
543
|
-
grpc_call_stack_destroy(
|
544
|
-
GRPC_CLOSURE_INIT(&c->
|
793
|
+
grpc_error_handle status_error = c->status_error_.get();
|
794
|
+
grpc_error_get_status(status_error, c->send_deadline(),
|
795
|
+
&c->final_info_.final_status, nullptr, nullptr,
|
796
|
+
&(c->final_info_.error_string));
|
797
|
+
c->status_error_.set(absl::OkStatus());
|
798
|
+
c->final_info_.stats.latency =
|
799
|
+
gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time_);
|
800
|
+
grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
|
801
|
+
GRPC_CLOSURE_INIT(&c->release_call_, ReleaseCall, c,
|
545
802
|
grpc_schedule_on_exec_ctx));
|
546
803
|
}
|
547
804
|
|
548
|
-
void
|
549
|
-
|
550
|
-
void grpc_call_unref(grpc_call* c) {
|
551
|
-
if (GPR_LIKELY(!c->ext_ref.Unref())) return;
|
552
|
-
|
553
|
-
GPR_TIMER_SCOPE("grpc_call_unref", 0);
|
805
|
+
void FilterStackCall::ExternalUnref() {
|
806
|
+
if (GPR_LIKELY(!ext_ref_.Unref())) return;
|
554
807
|
|
555
|
-
|
556
|
-
|
557
|
-
grpc_core::ExecCtx exec_ctx;
|
808
|
+
ApplicationCallbackExecCtx callback_exec_ctx;
|
809
|
+
ExecCtx exec_ctx;
|
558
810
|
|
559
|
-
GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (
|
811
|
+
GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (this));
|
560
812
|
|
561
|
-
|
562
|
-
parent_call* pc = get_parent_call(cc->parent);
|
563
|
-
gpr_mu_lock(&pc->child_list_mu);
|
564
|
-
if (c == pc->first_child) {
|
565
|
-
pc->first_child = cc->sibling_next;
|
566
|
-
if (c == pc->first_child) {
|
567
|
-
pc->first_child = nullptr;
|
568
|
-
}
|
569
|
-
}
|
570
|
-
cc->sibling_prev->child->sibling_next = cc->sibling_next;
|
571
|
-
cc->sibling_next->child->sibling_prev = cc->sibling_prev;
|
572
|
-
gpr_mu_unlock(&pc->child_list_mu);
|
573
|
-
GRPC_CALL_INTERNAL_UNREF(cc->parent, "child");
|
574
|
-
}
|
813
|
+
MaybeUnpublishFromParent();
|
575
814
|
|
576
|
-
GPR_ASSERT(!
|
577
|
-
|
578
|
-
bool cancel = gpr_atm_acq_load(&
|
579
|
-
gpr_atm_acq_load(&c->received_final_op_atm) == 0;
|
815
|
+
GPR_ASSERT(!destroy_called_);
|
816
|
+
destroy_called_ = true;
|
817
|
+
bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
|
580
818
|
if (cancel) {
|
581
|
-
|
819
|
+
CancelWithError(absl::CancelledError());
|
582
820
|
} else {
|
583
821
|
// Unset the call combiner cancellation closure. This has the
|
584
822
|
// effect of scheduling the previously set cancellation closure, if
|
585
823
|
// any, so that it can release any internal references it may be
|
586
824
|
// holding to the call stack.
|
587
|
-
|
825
|
+
call_combiner_.SetNotifyOnCancel(nullptr);
|
588
826
|
}
|
589
|
-
|
590
|
-
}
|
591
|
-
|
592
|
-
grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
|
593
|
-
GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved));
|
594
|
-
GPR_ASSERT(!reserved);
|
595
|
-
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
596
|
-
grpc_core::ExecCtx exec_ctx;
|
597
|
-
cancel_with_error(call, GRPC_ERROR_CANCELLED);
|
598
|
-
return GRPC_CALL_OK;
|
599
|
-
}
|
600
|
-
|
601
|
-
// This is called via the call combiner to start sending a batch down
|
602
|
-
// the filter stack.
|
603
|
-
static void execute_batch_in_call_combiner(void* arg,
|
604
|
-
grpc_error_handle /*ignored*/) {
|
605
|
-
GPR_TIMER_SCOPE("execute_batch_in_call_combiner", 0);
|
606
|
-
grpc_transport_stream_op_batch* batch =
|
607
|
-
static_cast<grpc_transport_stream_op_batch*>(arg);
|
608
|
-
grpc_call* call = static_cast<grpc_call*>(batch->handler_private.extra_arg);
|
609
|
-
grpc_call_element* elem = CALL_ELEM_FROM_CALL(call, 0);
|
610
|
-
GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
|
611
|
-
elem->filter->start_transport_stream_op_batch(elem, batch);
|
827
|
+
InternalUnref("destroy");
|
612
828
|
}
|
613
829
|
|
614
830
|
// start_batch_closure points to a caller-allocated closure to be used
|
615
831
|
// for entering the call combiner.
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
832
|
+
void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
|
833
|
+
grpc_closure* start_batch_closure) {
|
834
|
+
// This is called via the call combiner to start sending a batch down
|
835
|
+
// the filter stack.
|
836
|
+
auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
|
837
|
+
grpc_transport_stream_op_batch* batch =
|
838
|
+
static_cast<grpc_transport_stream_op_batch*>(arg);
|
839
|
+
auto* call =
|
840
|
+
static_cast<FilterStackCall*>(batch->handler_private.extra_arg);
|
841
|
+
grpc_call_element* elem = call->call_elem(0);
|
842
|
+
GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
|
843
|
+
elem->filter->start_transport_stream_op_batch(elem, batch);
|
844
|
+
};
|
845
|
+
batch->handler_private.extra_arg = this;
|
620
846
|
GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
|
621
847
|
grpc_schedule_on_exec_ctx);
|
622
|
-
GRPC_CALL_COMBINER_START(
|
623
|
-
|
624
|
-
}
|
625
|
-
|
626
|
-
char* grpc_call_get_peer(grpc_call* call) {
|
627
|
-
char* peer_string =
|
628
|
-
reinterpret_cast<char*>(gpr_atm_acq_load(&call->peer_string));
|
629
|
-
if (peer_string != nullptr) return gpr_strdup(peer_string);
|
630
|
-
peer_string = grpc_channel_get_target(call->channel);
|
631
|
-
if (peer_string != nullptr) return peer_string;
|
632
|
-
return gpr_strdup("unknown");
|
633
|
-
}
|
634
|
-
|
635
|
-
grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element) {
|
636
|
-
return CALL_FROM_TOP_ELEM(surface_element);
|
637
|
-
}
|
638
|
-
|
639
|
-
/*******************************************************************************
|
640
|
-
* CANCELLATION
|
641
|
-
*/
|
642
|
-
|
643
|
-
grpc_call_error grpc_call_cancel_with_status(grpc_call* c,
|
644
|
-
grpc_status_code status,
|
645
|
-
const char* description,
|
646
|
-
void* reserved) {
|
647
|
-
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
648
|
-
grpc_core::ExecCtx exec_ctx;
|
649
|
-
GRPC_API_TRACE(
|
650
|
-
"grpc_call_cancel_with_status("
|
651
|
-
"c=%p, status=%d, description=%s, reserved=%p)",
|
652
|
-
4, (c, (int)status, description, reserved));
|
653
|
-
GPR_ASSERT(reserved == nullptr);
|
654
|
-
cancel_with_status(c, status, description);
|
655
|
-
return GRPC_CALL_OK;
|
848
|
+
GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
|
849
|
+
absl::OkStatus(), "executing batch");
|
656
850
|
}
|
657
851
|
|
658
|
-
|
659
|
-
|
852
|
+
namespace {
|
853
|
+
struct CancelState {
|
854
|
+
FilterStackCall* call;
|
660
855
|
grpc_closure start_batch;
|
661
856
|
grpc_closure finish_batch;
|
662
857
|
};
|
858
|
+
} // namespace
|
859
|
+
|
663
860
|
// The on_complete callback used when sending a cancel_stream batch down
|
664
861
|
// the filter stack. Yields the call combiner when the batch is done.
|
665
862
|
static void done_termination(void* arg, grpc_error_handle /*error*/) {
|
666
|
-
|
667
|
-
GRPC_CALL_COMBINER_STOP(
|
863
|
+
CancelState* state = static_cast<CancelState*>(arg);
|
864
|
+
GRPC_CALL_COMBINER_STOP(state->call->call_combiner(),
|
668
865
|
"on_complete for cancel_stream op");
|
669
|
-
|
670
|
-
|
866
|
+
state->call->InternalUnref("termination");
|
867
|
+
delete state;
|
671
868
|
}
|
672
869
|
|
673
|
-
|
674
|
-
if (!gpr_atm_rel_cas(&
|
675
|
-
GRPC_ERROR_UNREF(error);
|
870
|
+
void FilterStackCall::CancelWithError(grpc_error_handle error) {
|
871
|
+
if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
|
676
872
|
return;
|
677
873
|
}
|
678
|
-
|
874
|
+
ClearPeerString();
|
875
|
+
InternalRef("termination");
|
679
876
|
// Inform the call combiner of the cancellation, so that it can cancel
|
680
877
|
// any in-flight asynchronous actions that may be holding the call
|
681
878
|
// combiner. This ensures that the cancel_stream batch can be sent
|
682
879
|
// down the filter stack in a timely manner.
|
683
|
-
|
684
|
-
|
685
|
-
state->call =
|
880
|
+
call_combiner_.Cancel(error);
|
881
|
+
CancelState* state = new CancelState;
|
882
|
+
state->call = this;
|
686
883
|
GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
|
687
884
|
grpc_schedule_on_exec_ctx);
|
688
885
|
grpc_transport_stream_op_batch* op =
|
689
886
|
grpc_make_transport_stream_op(&state->finish_batch);
|
690
887
|
op->cancel_stream = true;
|
691
888
|
op->payload->cancel_stream.cancel_error = error;
|
692
|
-
|
693
|
-
}
|
694
|
-
|
695
|
-
void grpc_call_cancel_internal(grpc_call* call) {
|
696
|
-
cancel_with_error(call, GRPC_ERROR_CANCELLED);
|
697
|
-
}
|
698
|
-
|
699
|
-
static grpc_error_handle error_from_status(grpc_status_code status,
|
700
|
-
const char* description) {
|
701
|
-
// copying 'description' is needed to ensure the grpc_call_cancel_with_status
|
702
|
-
// guarantee that can be short-lived.
|
703
|
-
return grpc_error_set_int(
|
704
|
-
grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description),
|
705
|
-
GRPC_ERROR_STR_GRPC_MESSAGE, description),
|
706
|
-
GRPC_ERROR_INT_GRPC_STATUS, status);
|
707
|
-
}
|
708
|
-
|
709
|
-
static void cancel_with_status(grpc_call* c, grpc_status_code status,
|
710
|
-
const char* description) {
|
711
|
-
cancel_with_error(c, error_from_status(status, description));
|
889
|
+
ExecuteBatch(op, &state->start_batch);
|
712
890
|
}
|
713
891
|
|
714
|
-
|
892
|
+
void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
|
715
893
|
if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
|
716
|
-
gpr_log(GPR_DEBUG, "set_final_status %s",
|
717
|
-
gpr_log(GPR_DEBUG, "%s",
|
894
|
+
gpr_log(GPR_DEBUG, "set_final_status %s", is_client() ? "CLI" : "SVR");
|
895
|
+
gpr_log(GPR_DEBUG, "%s", StatusToString(error).c_str());
|
718
896
|
}
|
719
|
-
if (
|
897
|
+
if (is_client()) {
|
720
898
|
std::string status_details;
|
721
|
-
grpc_error_get_status(error,
|
722
|
-
|
723
|
-
|
724
|
-
*
|
899
|
+
grpc_error_get_status(error, send_deadline(), final_op_.client.status,
|
900
|
+
&status_details, nullptr,
|
901
|
+
final_op_.client.error_string);
|
902
|
+
*final_op_.client.status_details =
|
725
903
|
grpc_slice_from_cpp_string(std::move(status_details));
|
726
|
-
|
727
|
-
|
728
|
-
grpc_core::channelz::ChannelNode* channelz_channel =
|
729
|
-
grpc_channel_get_channelz_node(call->channel);
|
904
|
+
status_error_.set(error);
|
905
|
+
channelz::ChannelNode* channelz_channel = channel()->channelz_node();
|
730
906
|
if (channelz_channel != nullptr) {
|
731
|
-
if (*
|
907
|
+
if (*final_op_.client.status != GRPC_STATUS_OK) {
|
732
908
|
channelz_channel->RecordCallFailed();
|
733
909
|
} else {
|
734
910
|
channelz_channel->RecordCallSucceeded();
|
735
911
|
}
|
736
912
|
}
|
737
913
|
} else {
|
738
|
-
*
|
739
|
-
error
|
740
|
-
|
741
|
-
|
914
|
+
*final_op_.server.cancelled =
|
915
|
+
!error.ok() || !sent_server_trailing_metadata_;
|
916
|
+
channelz::ServerNode* channelz_node =
|
917
|
+
final_op_.server.core_server->channelz_node();
|
742
918
|
if (channelz_node != nullptr) {
|
743
|
-
if (*
|
919
|
+
if (*final_op_.server.cancelled || !status_error_.ok()) {
|
744
920
|
channelz_node->RecordCallFailed();
|
745
921
|
} else {
|
746
922
|
channelz_node->RecordCallSucceeded();
|
747
923
|
}
|
748
924
|
}
|
749
|
-
GRPC_ERROR_UNREF(error);
|
750
925
|
}
|
751
926
|
}
|
752
927
|
|
753
|
-
|
754
|
-
|
755
|
-
|
756
|
-
|
757
|
-
|
758
|
-
|
759
|
-
return call->incoming_compression_algorithm;
|
760
|
-
}
|
761
|
-
|
762
|
-
uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
|
763
|
-
uint32_t flags;
|
764
|
-
flags = call->test_only_last_message_flags;
|
765
|
-
return flags;
|
766
|
-
}
|
767
|
-
|
768
|
-
uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) {
|
769
|
-
return call->encodings_accepted_by_peer.ToLegacyBitmask();
|
770
|
-
}
|
771
|
-
|
772
|
-
static int prepare_application_metadata(grpc_call* call, int count,
|
773
|
-
grpc_metadata* metadata,
|
774
|
-
int is_trailing) {
|
775
|
-
int i;
|
776
|
-
grpc_metadata_batch* batch = is_trailing ? &call->send_trailing_metadata
|
777
|
-
: &call->send_initial_metadata;
|
778
|
-
for (i = 0; i < count; i++) {
|
928
|
+
bool FilterStackCall::PrepareApplicationMetadata(size_t count,
|
929
|
+
grpc_metadata* metadata,
|
930
|
+
bool is_trailing) {
|
931
|
+
grpc_metadata_batch* batch =
|
932
|
+
is_trailing ? &send_trailing_metadata_ : &send_initial_metadata_;
|
933
|
+
for (size_t i = 0; i < count; i++) {
|
779
934
|
grpc_metadata* md = &metadata[i];
|
780
935
|
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
781
936
|
grpc_validate_header_key_is_legal(md->key))) {
|
782
|
-
return
|
937
|
+
return false;
|
783
938
|
} else if (!grpc_is_binary_header_internal(md->key) &&
|
784
939
|
!GRPC_LOG_IF_ERROR(
|
785
940
|
"validate_metadata",
|
786
941
|
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
787
|
-
return
|
942
|
+
return false;
|
788
943
|
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
789
944
|
// HTTP2 hpack encoding has a maximum limit.
|
790
|
-
return
|
945
|
+
return false;
|
946
|
+
} else if (grpc_slice_str_cmp(md->key, "content-length") == 0) {
|
947
|
+
// Filter "content-length metadata"
|
948
|
+
continue;
|
791
949
|
}
|
792
|
-
batch->Append(
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
.c_str());
|
801
|
-
});
|
950
|
+
batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
|
951
|
+
[md](absl::string_view error, const Slice& value) {
|
952
|
+
gpr_log(GPR_DEBUG, "Append error: %s",
|
953
|
+
absl::StrCat("key=", StringViewFromSlice(md->key),
|
954
|
+
" error=", error,
|
955
|
+
" value=", value.as_string_view())
|
956
|
+
.c_str());
|
957
|
+
});
|
802
958
|
}
|
803
959
|
|
804
|
-
return
|
960
|
+
return true;
|
805
961
|
}
|
806
962
|
|
807
963
|
namespace {
|
@@ -809,7 +965,7 @@ class PublishToAppEncoder {
|
|
809
965
|
public:
|
810
966
|
explicit PublishToAppEncoder(grpc_metadata_array* dest) : dest_(dest) {}
|
811
967
|
|
812
|
-
void Encode(const
|
968
|
+
void Encode(const Slice& key, const Slice& value) {
|
813
969
|
Append(key.c_slice(), value.c_slice());
|
814
970
|
}
|
815
971
|
|
@@ -819,36 +975,34 @@ class PublishToAppEncoder {
|
|
819
975
|
template <typename Which>
|
820
976
|
void Encode(Which, const typename Which::ValueType&) {}
|
821
977
|
|
822
|
-
void Encode(
|
823
|
-
Append(
|
978
|
+
void Encode(UserAgentMetadata, const Slice& slice) {
|
979
|
+
Append(UserAgentMetadata::key(), slice);
|
824
980
|
}
|
825
981
|
|
826
|
-
void Encode(
|
827
|
-
Append(
|
982
|
+
void Encode(HostMetadata, const Slice& slice) {
|
983
|
+
Append(HostMetadata::key(), slice);
|
828
984
|
}
|
829
985
|
|
830
|
-
void Encode(
|
831
|
-
Append(
|
986
|
+
void Encode(GrpcPreviousRpcAttemptsMetadata, uint32_t count) {
|
987
|
+
Append(GrpcPreviousRpcAttemptsMetadata::key(), count);
|
832
988
|
}
|
833
989
|
|
834
|
-
void Encode(
|
835
|
-
|
836
|
-
Append(grpc_core::GrpcRetryPushbackMsMetadata::key(), count.millis());
|
990
|
+
void Encode(GrpcRetryPushbackMsMetadata, Duration count) {
|
991
|
+
Append(GrpcRetryPushbackMsMetadata::key(), count.millis());
|
837
992
|
}
|
838
993
|
|
839
|
-
void Encode(
|
840
|
-
Append(
|
994
|
+
void Encode(LbTokenMetadata, const Slice& slice) {
|
995
|
+
Append(LbTokenMetadata::key(), slice);
|
841
996
|
}
|
842
997
|
|
843
998
|
private:
|
844
999
|
void Append(absl::string_view key, int64_t value) {
|
845
|
-
Append(
|
846
|
-
|
1000
|
+
Append(StaticSlice::FromStaticString(key).c_slice(),
|
1001
|
+
Slice::FromInt64(value).c_slice());
|
847
1002
|
}
|
848
1003
|
|
849
|
-
void Append(absl::string_view key, const
|
850
|
-
Append(
|
851
|
-
value.c_slice());
|
1004
|
+
void Append(absl::string_view key, const Slice& value) {
|
1005
|
+
Append(StaticSlice::FromStaticString(key).c_slice(), value.c_slice());
|
852
1006
|
}
|
853
1007
|
|
854
1008
|
void Append(grpc_slice key, grpc_slice value) {
|
@@ -861,14 +1015,13 @@ class PublishToAppEncoder {
|
|
861
1015
|
};
|
862
1016
|
} // namespace
|
863
1017
|
|
864
|
-
|
865
|
-
|
1018
|
+
void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
|
1019
|
+
bool is_trailing) {
|
866
1020
|
if (b->count() == 0) return;
|
867
|
-
if (!
|
868
|
-
if (is_trailing &&
|
869
|
-
GPR_TIMER_SCOPE("publish_app_metadata", 0);
|
1021
|
+
if (!is_client() && is_trailing) return;
|
1022
|
+
if (is_trailing && buffered_metadata_[1] == nullptr) return;
|
870
1023
|
grpc_metadata_array* dest;
|
871
|
-
dest =
|
1024
|
+
dest = buffered_metadata_[is_trailing];
|
872
1025
|
if (dest->count + b->count() > dest->capacity) {
|
873
1026
|
dest->capacity =
|
874
1027
|
std::max(dest->capacity + b->count(), dest->capacity * 3 / 2);
|
@@ -879,85 +1032,69 @@ static void publish_app_metadata(grpc_call* call, grpc_metadata_batch* b,
|
|
879
1032
|
b->Encode(&encoder);
|
880
1033
|
}
|
881
1034
|
|
882
|
-
|
883
|
-
|
884
|
-
b->Take(
|
885
|
-
|
886
|
-
b->Take(
|
887
|
-
.value_or(
|
888
|
-
|
1035
|
+
void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
|
1036
|
+
incoming_compression_algorithm_ =
|
1037
|
+
b->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
1038
|
+
encodings_accepted_by_peer_ =
|
1039
|
+
b->Take(GrpcAcceptEncodingMetadata())
|
1040
|
+
.value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
|
1041
|
+
PublishAppMetadata(b, false);
|
889
1042
|
}
|
890
1043
|
|
891
|
-
|
892
|
-
|
893
|
-
|
894
|
-
|
895
|
-
set_final_status(call, batch_error);
|
1044
|
+
void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
1045
|
+
grpc_error_handle batch_error) {
|
1046
|
+
if (!batch_error.ok()) {
|
1047
|
+
SetFinalStatus(batch_error);
|
896
1048
|
} else {
|
897
1049
|
absl::optional<grpc_status_code> grpc_status =
|
898
|
-
b->Take(
|
1050
|
+
b->Take(GrpcStatusMetadata());
|
899
1051
|
if (grpc_status.has_value()) {
|
900
1052
|
grpc_status_code status_code = *grpc_status;
|
901
|
-
grpc_error_handle error
|
1053
|
+
grpc_error_handle error;
|
902
1054
|
if (status_code != GRPC_STATUS_OK) {
|
903
|
-
char* peer =
|
1055
|
+
char* peer = GetPeer();
|
904
1056
|
error = grpc_error_set_int(
|
905
|
-
|
906
|
-
|
907
|
-
GRPC_ERROR_INT_GRPC_STATUS, static_cast<intptr_t>(status_code));
|
1057
|
+
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ", peer)),
|
1058
|
+
StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
|
908
1059
|
gpr_free(peer);
|
909
1060
|
}
|
910
|
-
auto grpc_message = b->Take(
|
1061
|
+
auto grpc_message = b->Take(GrpcMessageMetadata());
|
911
1062
|
if (grpc_message.has_value()) {
|
912
|
-
error = grpc_error_set_str(error,
|
1063
|
+
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
|
913
1064
|
grpc_message->as_string_view());
|
914
|
-
} else if (error
|
915
|
-
error = grpc_error_set_str(error,
|
1065
|
+
} else if (!error.ok()) {
|
1066
|
+
error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
|
916
1067
|
}
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
set_final_status(call, GRPC_ERROR_NONE);
|
1068
|
+
SetFinalStatus(error);
|
1069
|
+
} else if (!is_client()) {
|
1070
|
+
SetFinalStatus(absl::OkStatus());
|
921
1071
|
} else {
|
922
1072
|
gpr_log(GPR_DEBUG,
|
923
1073
|
"Received trailing metadata with no error and no status");
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNKNOWN));
|
1074
|
+
SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
|
1075
|
+
StatusIntProperty::kRpcStatus,
|
1076
|
+
GRPC_STATUS_UNKNOWN));
|
928
1077
|
}
|
929
1078
|
}
|
930
|
-
|
931
|
-
}
|
932
|
-
|
933
|
-
grpc_core::Arena* grpc_call_get_arena(grpc_call* call) { return call->arena; }
|
934
|
-
|
935
|
-
grpc_call_stack* grpc_call_get_call_stack(grpc_call* call) {
|
936
|
-
return CALL_STACK_FROM_CALL(call);
|
1079
|
+
PublishAppMetadata(b, true);
|
937
1080
|
}
|
938
1081
|
|
939
|
-
|
940
|
-
|
941
|
-
|
942
|
-
|
943
|
-
static bool are_write_flags_valid(uint32_t flags) {
|
944
|
-
/* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */
|
1082
|
+
namespace {
|
1083
|
+
bool AreWriteFlagsValid(uint32_t flags) {
|
1084
|
+
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
945
1085
|
const uint32_t allowed_write_positions =
|
946
1086
|
(GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
|
947
1087
|
const uint32_t invalid_positions = ~allowed_write_positions;
|
948
1088
|
return !(flags & invalid_positions);
|
949
1089
|
}
|
950
1090
|
|
951
|
-
|
952
|
-
|
1091
|
+
bool AreInitialMetadataFlagsValid(uint32_t flags) {
|
1092
|
+
// check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
|
953
1093
|
uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
|
954
|
-
if (!is_client) {
|
955
|
-
invalid_positions |= GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
|
956
|
-
}
|
957
1094
|
return !(flags & invalid_positions);
|
958
1095
|
}
|
959
1096
|
|
960
|
-
|
1097
|
+
size_t BatchSlotForOp(grpc_op_type type) {
|
961
1098
|
switch (type) {
|
962
1099
|
case GRPC_OP_SEND_INITIAL_METADATA:
|
963
1100
|
return 0;
|
@@ -976,374 +1113,293 @@ static size_t batch_slot_for_op(grpc_op_type type) {
|
|
976
1113
|
}
|
977
1114
|
GPR_UNREACHABLE_CODE(return 123456789);
|
978
1115
|
}
|
1116
|
+
} // namespace
|
979
1117
|
|
980
|
-
|
981
|
-
|
982
|
-
size_t slot_idx =
|
983
|
-
|
984
|
-
|
1118
|
+
FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
|
1119
|
+
const grpc_op* ops) {
|
1120
|
+
size_t slot_idx = BatchSlotForOp(ops[0].op);
|
1121
|
+
BatchControl** pslot = &active_batches_[slot_idx];
|
1122
|
+
BatchControl* bctl;
|
985
1123
|
if (*pslot != nullptr) {
|
986
1124
|
bctl = *pslot;
|
987
|
-
if (bctl->
|
1125
|
+
if (bctl->call_ != nullptr) {
|
988
1126
|
return nullptr;
|
989
1127
|
}
|
990
|
-
bctl->~
|
991
|
-
bctl->
|
992
|
-
new (&bctl->
|
1128
|
+
bctl->~BatchControl();
|
1129
|
+
bctl->op_ = {};
|
1130
|
+
new (&bctl->batch_error_) AtomicError();
|
993
1131
|
} else {
|
994
|
-
bctl =
|
1132
|
+
bctl = arena()->New<BatchControl>();
|
995
1133
|
*pslot = bctl;
|
996
1134
|
}
|
997
|
-
bctl->
|
998
|
-
bctl->
|
1135
|
+
bctl->call_ = this;
|
1136
|
+
bctl->op_.payload = &stream_op_payload_;
|
999
1137
|
return bctl;
|
1000
1138
|
}
|
1001
1139
|
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
}
|
1009
|
-
|
1010
|
-
static void reset_batch_errors(batch_control* bctl) {
|
1011
|
-
bctl->batch_error.set(GRPC_ERROR_NONE);
|
1012
|
-
}
|
1013
|
-
|
1014
|
-
static void post_batch_completion(batch_control* bctl) {
|
1015
|
-
grpc_call* next_child_call;
|
1016
|
-
grpc_call* call = bctl->call;
|
1017
|
-
grpc_error_handle error = GRPC_ERROR_REF(bctl->batch_error.get());
|
1140
|
+
void FilterStackCall::BatchControl::PostCompletion() {
|
1141
|
+
FilterStackCall* call = call_;
|
1142
|
+
grpc_error_handle error = batch_error_.get();
|
1143
|
+
if (grpc_call_trace.enabled()) {
|
1144
|
+
gpr_log(GPR_DEBUG, "tag:%p batch_error=%s", completion_data_.notify_tag.tag,
|
1145
|
+
error.ToString().c_str());
|
1146
|
+
}
|
1018
1147
|
|
1019
|
-
if (
|
1020
|
-
call->
|
1148
|
+
if (op_.send_initial_metadata) {
|
1149
|
+
call->send_initial_metadata_.Clear();
|
1021
1150
|
}
|
1022
|
-
if (
|
1023
|
-
if (
|
1024
|
-
error == GRPC_ERROR_NONE) {
|
1151
|
+
if (op_.send_message) {
|
1152
|
+
if (op_.payload->send_message.stream_write_closed && error.ok()) {
|
1025
1153
|
error = grpc_error_add_child(
|
1026
|
-
error,
|
1154
|
+
error, GRPC_ERROR_CREATE(
|
1027
1155
|
"Attempt to send message after stream was closed."));
|
1028
1156
|
}
|
1029
|
-
call->
|
1030
|
-
|
1031
|
-
|
1032
|
-
|
1033
|
-
|
1034
|
-
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1040
|
-
|
1041
|
-
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1050
|
-
|
1051
|
-
|
1052
|
-
|
1053
|
-
gpr_mu_unlock(&pc->child_list_mu);
|
1054
|
-
}
|
1055
|
-
GRPC_ERROR_UNREF(error);
|
1056
|
-
error = GRPC_ERROR_NONE;
|
1057
|
-
}
|
1058
|
-
if (error != GRPC_ERROR_NONE && bctl->op.recv_message &&
|
1059
|
-
*call->receiving_buffer != nullptr) {
|
1060
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer);
|
1061
|
-
*call->receiving_buffer = nullptr;
|
1062
|
-
}
|
1063
|
-
reset_batch_errors(bctl);
|
1064
|
-
|
1065
|
-
if (bctl->completion_data.notify_tag.is_closure) {
|
1066
|
-
/* unrefs error */
|
1067
|
-
bctl->call = nullptr;
|
1068
|
-
grpc_core::Closure::Run(
|
1069
|
-
DEBUG_LOCATION,
|
1070
|
-
static_cast<grpc_closure*>(bctl->completion_data.notify_tag.tag),
|
1071
|
-
error);
|
1072
|
-
GRPC_CALL_INTERNAL_UNREF(call, "completion");
|
1157
|
+
call->sending_message_ = false;
|
1158
|
+
call->send_slice_buffer_.Clear();
|
1159
|
+
}
|
1160
|
+
if (op_.send_trailing_metadata) {
|
1161
|
+
call->send_trailing_metadata_.Clear();
|
1162
|
+
}
|
1163
|
+
if (op_.recv_trailing_metadata) {
|
1164
|
+
// propagate cancellation to any interested children
|
1165
|
+
gpr_atm_rel_store(&call->received_final_op_atm_, 1);
|
1166
|
+
call->PropagateCancellationToChildren();
|
1167
|
+
error = absl::OkStatus();
|
1168
|
+
}
|
1169
|
+
if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
|
1170
|
+
grpc_byte_buffer_destroy(*call->receiving_buffer_);
|
1171
|
+
*call->receiving_buffer_ = nullptr;
|
1172
|
+
}
|
1173
|
+
batch_error_.set(absl::OkStatus());
|
1174
|
+
|
1175
|
+
if (completion_data_.notify_tag.is_closure) {
|
1176
|
+
call_ = nullptr;
|
1177
|
+
Closure::Run(DEBUG_LOCATION,
|
1178
|
+
static_cast<grpc_closure*>(completion_data_.notify_tag.tag),
|
1179
|
+
error);
|
1180
|
+
call->InternalUnref("completion");
|
1073
1181
|
} else {
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
post_batch_completion(bctl);
|
1084
|
-
}
|
1085
|
-
}
|
1086
|
-
|
1087
|
-
static void continue_receiving_slices(batch_control* bctl) {
|
1088
|
-
grpc_error_handle error;
|
1089
|
-
grpc_call* call = bctl->call;
|
1090
|
-
for (;;) {
|
1091
|
-
size_t remaining = call->receiving_stream->length() -
|
1092
|
-
(*call->receiving_buffer)->data.raw.slice_buffer.length;
|
1093
|
-
if (remaining == 0) {
|
1094
|
-
call->receiving_message = false;
|
1095
|
-
call->receiving_stream.reset();
|
1096
|
-
finish_batch_step(bctl);
|
1097
|
-
return;
|
1098
|
-
}
|
1099
|
-
if (call->receiving_stream->Next(remaining, &call->receiving_slice_ready)) {
|
1100
|
-
error = call->receiving_stream->Pull(&call->receiving_slice);
|
1101
|
-
if (error == GRPC_ERROR_NONE) {
|
1102
|
-
grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
|
1103
|
-
call->receiving_slice);
|
1104
|
-
} else {
|
1105
|
-
call->receiving_stream.reset();
|
1106
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer);
|
1107
|
-
*call->receiving_buffer = nullptr;
|
1108
|
-
call->receiving_message = false;
|
1109
|
-
finish_batch_step(bctl);
|
1110
|
-
GRPC_ERROR_UNREF(error);
|
1111
|
-
return;
|
1112
|
-
}
|
1113
|
-
} else {
|
1114
|
-
return;
|
1115
|
-
}
|
1182
|
+
grpc_cq_end_op(
|
1183
|
+
call->cq_, completion_data_.notify_tag.tag, error,
|
1184
|
+
[](void* user_data, grpc_cq_completion* /*storage*/) {
|
1185
|
+
BatchControl* bctl = static_cast<BatchControl*>(user_data);
|
1186
|
+
Call* call = bctl->call_;
|
1187
|
+
bctl->call_ = nullptr;
|
1188
|
+
call->InternalUnref("completion");
|
1189
|
+
},
|
1190
|
+
this, &completion_data_.cq_completion);
|
1116
1191
|
}
|
1117
1192
|
}
|
1118
1193
|
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
bool release_error = false;
|
1123
|
-
|
1124
|
-
if (error == GRPC_ERROR_NONE) {
|
1125
|
-
grpc_slice slice;
|
1126
|
-
error = call->receiving_stream->Pull(&slice);
|
1127
|
-
if (error == GRPC_ERROR_NONE) {
|
1128
|
-
grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer,
|
1129
|
-
slice);
|
1130
|
-
continue_receiving_slices(bctl);
|
1131
|
-
} else {
|
1132
|
-
/* Error returned by ByteStream::Pull() needs to be released manually */
|
1133
|
-
release_error = true;
|
1134
|
-
}
|
1135
|
-
}
|
1136
|
-
|
1137
|
-
if (error != GRPC_ERROR_NONE) {
|
1138
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
|
1139
|
-
GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
|
1140
|
-
}
|
1141
|
-
call->receiving_stream.reset();
|
1142
|
-
grpc_byte_buffer_destroy(*call->receiving_buffer);
|
1143
|
-
*call->receiving_buffer = nullptr;
|
1144
|
-
call->receiving_message = false;
|
1145
|
-
finish_batch_step(bctl);
|
1146
|
-
if (release_error) {
|
1147
|
-
GRPC_ERROR_UNREF(error);
|
1148
|
-
}
|
1194
|
+
void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
|
1195
|
+
if (GPR_UNLIKELY(completed_batch_step(op))) {
|
1196
|
+
PostCompletion();
|
1149
1197
|
}
|
1150
1198
|
}
|
1151
1199
|
|
1152
|
-
|
1153
|
-
|
1154
|
-
if (call->
|
1155
|
-
*call->
|
1156
|
-
call->
|
1157
|
-
|
1200
|
+
void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
|
1201
|
+
FilterStackCall* call = call_;
|
1202
|
+
if (!call->receiving_slice_buffer_.has_value()) {
|
1203
|
+
*call->receiving_buffer_ = nullptr;
|
1204
|
+
call->receiving_message_ = false;
|
1205
|
+
FinishStep(PendingOp::kRecvMessage);
|
1158
1206
|
} else {
|
1159
|
-
call->
|
1160
|
-
if ((call->
|
1161
|
-
(call->
|
1162
|
-
*call->
|
1163
|
-
nullptr, 0, call->
|
1207
|
+
call->test_only_last_message_flags_ = call->receiving_stream_flags_;
|
1208
|
+
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
1209
|
+
(call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
|
1210
|
+
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
|
1211
|
+
nullptr, 0, call->incoming_compression_algorithm_);
|
1164
1212
|
} else {
|
1165
|
-
*call->
|
1213
|
+
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
1166
1214
|
}
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1215
|
+
grpc_slice_buffer_move_into(
|
1216
|
+
call->receiving_slice_buffer_->c_slice_buffer(),
|
1217
|
+
&(*call->receiving_buffer_)->data.raw.slice_buffer);
|
1218
|
+
call->receiving_message_ = false;
|
1219
|
+
call->receiving_slice_buffer_.reset();
|
1220
|
+
FinishStep(PendingOp::kRecvMessage);
|
1170
1221
|
}
|
1171
1222
|
}
|
1172
1223
|
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1224
|
+
void FilterStackCall::BatchControl::ReceivingStreamReady(
|
1225
|
+
grpc_error_handle error) {
|
1226
|
+
if (grpc_call_trace.enabled()) {
|
1227
|
+
gpr_log(GPR_DEBUG,
|
1228
|
+
"tag:%p ReceivingStreamReady error=%s "
|
1229
|
+
"receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
|
1230
|
+
completion_data_.notify_tag.tag, error.ToString().c_str(),
|
1231
|
+
call_->receiving_slice_buffer_.has_value(),
|
1232
|
+
gpr_atm_no_barrier_load(&call_->recv_state_));
|
1233
|
+
}
|
1234
|
+
FilterStackCall* call = call_;
|
1235
|
+
if (!error.ok()) {
|
1236
|
+
call->receiving_slice_buffer_.reset();
|
1237
|
+
if (batch_error_.ok()) {
|
1238
|
+
batch_error_.set(error);
|
1180
1239
|
}
|
1181
|
-
|
1240
|
+
call->CancelWithError(error);
|
1182
1241
|
}
|
1183
|
-
|
1184
|
-
|
1185
|
-
|
1186
|
-
if (error
|
1187
|
-
!gpr_atm_rel_cas(&call->
|
1188
|
-
reinterpret_cast<gpr_atm>(
|
1189
|
-
|
1242
|
+
// If recv_state is kRecvNone, we will save the batch_control
|
1243
|
+
// object with rel_cas, and will not use it after the cas. Its corresponding
|
1244
|
+
// acq_load is in receiving_initial_metadata_ready()
|
1245
|
+
if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
|
1246
|
+
!gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
|
1247
|
+
reinterpret_cast<gpr_atm>(this))) {
|
1248
|
+
ProcessDataAfterMetadata();
|
1190
1249
|
}
|
1191
1250
|
}
|
1192
1251
|
|
1193
|
-
|
1194
|
-
|
1195
|
-
// before processing the received message.
|
1196
|
-
static void receiving_stream_ready_in_call_combiner(void* bctlp,
|
1197
|
-
grpc_error_handle error) {
|
1198
|
-
batch_control* bctl = static_cast<batch_control*>(bctlp);
|
1199
|
-
grpc_call* call = bctl->call;
|
1200
|
-
GRPC_CALL_COMBINER_STOP(&call->call_combiner, "recv_message_ready");
|
1201
|
-
receiving_stream_ready(bctlp, error);
|
1202
|
-
}
|
1203
|
-
|
1204
|
-
static void GPR_ATTRIBUTE_NOINLINE handle_compression_algorithm_disabled(
|
1205
|
-
grpc_call* call, grpc_compression_algorithm compression_algorithm) {
|
1252
|
+
void FilterStackCall::HandleCompressionAlgorithmDisabled(
|
1253
|
+
grpc_compression_algorithm compression_algorithm) {
|
1206
1254
|
const char* algo_name = nullptr;
|
1207
1255
|
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
1208
1256
|
std::string error_msg =
|
1209
1257
|
absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
|
1210
1258
|
gpr_log(GPR_ERROR, "%s", error_msg.c_str());
|
1211
|
-
|
1259
|
+
CancelWithStatus(GRPC_STATUS_UNIMPLEMENTED, error_msg.c_str());
|
1212
1260
|
}
|
1213
1261
|
|
1214
|
-
|
1215
|
-
|
1262
|
+
void FilterStackCall::HandleCompressionAlgorithmNotAccepted(
|
1263
|
+
grpc_compression_algorithm compression_algorithm) {
|
1216
1264
|
const char* algo_name = nullptr;
|
1217
1265
|
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
1218
1266
|
gpr_log(GPR_ERROR,
|
1219
1267
|
"Compression algorithm ('%s') not present in the "
|
1220
1268
|
"accepted encodings (%s)",
|
1221
|
-
algo_name,
|
1269
|
+
algo_name,
|
1270
|
+
std::string(encodings_accepted_by_peer_.ToString()).c_str());
|
1222
1271
|
}
|
1223
1272
|
|
1224
|
-
|
1225
|
-
|
1273
|
+
void FilterStackCall::BatchControl::ValidateFilteredMetadata() {
|
1274
|
+
FilterStackCall* call = call_;
|
1226
1275
|
|
1227
1276
|
const grpc_compression_options compression_options =
|
1228
|
-
|
1277
|
+
call->channel()->compression_options();
|
1229
1278
|
const grpc_compression_algorithm compression_algorithm =
|
1230
|
-
call->
|
1231
|
-
if (GPR_UNLIKELY(!
|
1279
|
+
call->incoming_compression_algorithm_;
|
1280
|
+
if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
|
1232
1281
|
compression_options.enabled_algorithms_bitset)
|
1233
1282
|
.IsSet(compression_algorithm))) {
|
1234
|
-
|
1235
|
-
|
1283
|
+
// check if algorithm is supported by current channel config
|
1284
|
+
call->HandleCompressionAlgorithmDisabled(compression_algorithm);
|
1236
1285
|
}
|
1237
|
-
|
1238
|
-
GPR_DEBUG_ASSERT(call->
|
1286
|
+
// GRPC_COMPRESS_NONE is always set.
|
1287
|
+
GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
1239
1288
|
if (GPR_UNLIKELY(
|
1240
|
-
!call->
|
1289
|
+
!call->encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
1241
1290
|
if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
|
1242
|
-
|
1291
|
+
call->HandleCompressionAlgorithmNotAccepted(compression_algorithm);
|
1243
1292
|
}
|
1244
1293
|
}
|
1245
1294
|
}
|
1246
1295
|
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
grpc_call* call = bctl->call;
|
1296
|
+
void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
1297
|
+
grpc_error_handle error) {
|
1298
|
+
FilterStackCall* call = call_;
|
1251
1299
|
|
1252
|
-
GRPC_CALL_COMBINER_STOP(
|
1300
|
+
GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
|
1253
1301
|
|
1254
|
-
if (error
|
1255
|
-
grpc_metadata_batch* md = &call->
|
1256
|
-
|
1302
|
+
if (error.ok()) {
|
1303
|
+
grpc_metadata_batch* md = &call->recv_initial_metadata_;
|
1304
|
+
call->RecvInitialFilter(md);
|
1257
1305
|
|
1258
|
-
|
1259
|
-
|
1260
|
-
validate_filtered_metadata(bctl);
|
1306
|
+
// TODO(ctiller): this could be moved into recv_initial_filter now
|
1307
|
+
ValidateFilteredMetadata();
|
1261
1308
|
|
1262
|
-
absl::optional<
|
1263
|
-
|
1264
|
-
|
1265
|
-
call->send_deadline = *deadline;
|
1309
|
+
absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
|
1310
|
+
if (deadline.has_value() && !call->is_client()) {
|
1311
|
+
call_->set_send_deadline(*deadline);
|
1266
1312
|
}
|
1267
1313
|
} else {
|
1268
|
-
if (
|
1269
|
-
|
1314
|
+
if (batch_error_.ok()) {
|
1315
|
+
batch_error_.set(error);
|
1270
1316
|
}
|
1271
|
-
|
1317
|
+
call->CancelWithError(error);
|
1272
1318
|
}
|
1273
1319
|
|
1274
1320
|
grpc_closure* saved_rsr_closure = nullptr;
|
1275
1321
|
while (true) {
|
1276
|
-
gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->
|
1277
|
-
|
1322
|
+
gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
|
1323
|
+
// Should only receive initial metadata once
|
1278
1324
|
GPR_ASSERT(rsr_bctlp != 1);
|
1279
1325
|
if (rsr_bctlp == 0) {
|
1280
|
-
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
1284
|
-
|
1285
|
-
if (gpr_atm_no_barrier_cas(&call->
|
1286
|
-
|
1326
|
+
// We haven't seen initial metadata and messages before, thus initial
|
1327
|
+
// metadata is received first.
|
1328
|
+
// no_barrier_cas is used, as this function won't access the batch_control
|
1329
|
+
// object saved by receiving_stream_ready() if the initial metadata is
|
1330
|
+
// received first.
|
1331
|
+
if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
|
1332
|
+
kRecvInitialMetadataFirst)) {
|
1287
1333
|
break;
|
1288
1334
|
}
|
1289
1335
|
} else {
|
1290
|
-
|
1291
|
-
saved_rsr_closure =
|
1292
|
-
|
1293
|
-
|
1294
|
-
|
1336
|
+
// Already received messages
|
1337
|
+
saved_rsr_closure = GRPC_CLOSURE_CREATE(
|
1338
|
+
[](void* bctl, grpc_error_handle error) {
|
1339
|
+
static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
|
1340
|
+
},
|
1341
|
+
reinterpret_cast<BatchControl*>(rsr_bctlp),
|
1342
|
+
grpc_schedule_on_exec_ctx);
|
1343
|
+
// No need to modify recv_state
|
1295
1344
|
break;
|
1296
1345
|
}
|
1297
1346
|
}
|
1298
1347
|
if (saved_rsr_closure != nullptr) {
|
1299
|
-
|
1300
|
-
GRPC_ERROR_REF(error));
|
1348
|
+
Closure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
|
1301
1349
|
}
|
1302
1350
|
|
1303
|
-
|
1351
|
+
FinishStep(PendingOp::kRecvInitialMetadata);
|
1304
1352
|
}
|
1305
1353
|
|
1306
|
-
|
1307
|
-
|
1308
|
-
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
finish_batch_step(bctl);
|
1354
|
+
void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
|
1355
|
+
grpc_error_handle error) {
|
1356
|
+
GRPC_CALL_COMBINER_STOP(call_->call_combiner(),
|
1357
|
+
"recv_trailing_metadata_ready");
|
1358
|
+
grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
|
1359
|
+
call_->RecvTrailingFilter(md, error);
|
1360
|
+
FinishStep(PendingOp::kRecvTrailingMetadata);
|
1314
1361
|
}
|
1315
1362
|
|
1316
|
-
|
1317
|
-
|
1318
|
-
|
1319
|
-
|
1320
|
-
if (bctl->batch_error.ok()) {
|
1321
|
-
bctl->batch_error.set(error);
|
1363
|
+
void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
|
1364
|
+
GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "on_complete");
|
1365
|
+
if (batch_error_.ok()) {
|
1366
|
+
batch_error_.set(error);
|
1322
1367
|
}
|
1323
|
-
if (error
|
1324
|
-
|
1368
|
+
if (!error.ok()) {
|
1369
|
+
call_->CancelWithError(error);
|
1325
1370
|
}
|
1326
|
-
|
1371
|
+
FinishStep(PendingOp::kSends);
|
1327
1372
|
}
|
1328
1373
|
|
1329
|
-
|
1330
|
-
|
1374
|
+
namespace {
|
1375
|
+
void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag,
|
1376
|
+
bool is_notify_tag_closure) {
|
1377
|
+
if (!is_notify_tag_closure) {
|
1378
|
+
GPR_ASSERT(grpc_cq_begin_op(cq, notify_tag));
|
1379
|
+
grpc_cq_end_op(
|
1380
|
+
cq, notify_tag, absl::OkStatus(),
|
1381
|
+
[](void*, grpc_cq_completion* completion) { gpr_free(completion); },
|
1382
|
+
nullptr,
|
1383
|
+
static_cast<grpc_cq_completion*>(
|
1384
|
+
gpr_malloc(sizeof(grpc_cq_completion))));
|
1385
|
+
} else {
|
1386
|
+
Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
|
1387
|
+
absl::OkStatus());
|
1388
|
+
}
|
1331
1389
|
}
|
1390
|
+
} // namespace
|
1332
1391
|
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
GPR_TIMER_SCOPE("call_start_batch", 0);
|
1337
|
-
|
1392
|
+
grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
1393
|
+
void* notify_tag,
|
1394
|
+
bool is_notify_tag_closure) {
|
1338
1395
|
size_t i;
|
1339
1396
|
const grpc_op* op;
|
1340
|
-
|
1341
|
-
bool has_send_ops = false;
|
1342
|
-
int num_recv_ops = 0;
|
1397
|
+
BatchControl* bctl;
|
1343
1398
|
grpc_call_error error = GRPC_CALL_OK;
|
1344
1399
|
grpc_transport_stream_op_batch* stream_op;
|
1345
1400
|
grpc_transport_stream_op_batch_payload* stream_op_payload;
|
1346
1401
|
uint32_t seen_ops = 0;
|
1402
|
+
intptr_t pending_ops = 0;
|
1347
1403
|
|
1348
1404
|
for (i = 0; i < nops; i++) {
|
1349
1405
|
if (seen_ops & (1u << ops[i].op)) {
|
@@ -1352,36 +1408,35 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1352
1408
|
seen_ops |= (1u << ops[i].op);
|
1353
1409
|
}
|
1354
1410
|
|
1411
|
+
if (!is_client() &&
|
1412
|
+
(seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
|
1413
|
+
(seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
|
1414
|
+
gpr_log(GPR_ERROR,
|
1415
|
+
"******************* SEND_STATUS WITH RECV_MESSAGE "
|
1416
|
+
"*******************");
|
1417
|
+
return GRPC_CALL_ERROR;
|
1418
|
+
}
|
1419
|
+
|
1355
1420
|
GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
|
1356
1421
|
|
1357
1422
|
if (nops == 0) {
|
1358
|
-
|
1359
|
-
GPR_ASSERT(grpc_cq_begin_op(call->cq, notify_tag));
|
1360
|
-
grpc_cq_end_op(call->cq, notify_tag, GRPC_ERROR_NONE,
|
1361
|
-
free_no_op_completion, nullptr,
|
1362
|
-
static_cast<grpc_cq_completion*>(
|
1363
|
-
gpr_malloc(sizeof(grpc_cq_completion))));
|
1364
|
-
} else {
|
1365
|
-
grpc_core::Closure::Run(DEBUG_LOCATION,
|
1366
|
-
static_cast<grpc_closure*>(notify_tag),
|
1367
|
-
GRPC_ERROR_NONE);
|
1368
|
-
}
|
1423
|
+
EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
|
1369
1424
|
error = GRPC_CALL_OK;
|
1370
1425
|
goto done;
|
1371
1426
|
}
|
1372
1427
|
|
1373
|
-
bctl =
|
1428
|
+
bctl = ReuseOrAllocateBatchControl(ops);
|
1374
1429
|
if (bctl == nullptr) {
|
1375
1430
|
return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1376
1431
|
}
|
1377
|
-
bctl->
|
1378
|
-
bctl->
|
1432
|
+
bctl->completion_data_.notify_tag.tag = notify_tag;
|
1433
|
+
bctl->completion_data_.notify_tag.is_closure =
|
1379
1434
|
static_cast<uint8_t>(is_notify_tag_closure != 0);
|
1380
1435
|
|
1381
|
-
stream_op = &bctl->
|
1382
|
-
stream_op_payload = &
|
1436
|
+
stream_op = &bctl->op_;
|
1437
|
+
stream_op_payload = &stream_op_payload_;
|
1383
1438
|
|
1384
|
-
|
1439
|
+
// rewrite batch ops into a transport op
|
1385
1440
|
for (i = 0; i < nops; i++) {
|
1386
1441
|
op = &ops[i];
|
1387
1442
|
if (op->reserved != nullptr) {
|
@@ -1390,12 +1445,12 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1390
1445
|
}
|
1391
1446
|
switch (op->op) {
|
1392
1447
|
case GRPC_OP_SEND_INITIAL_METADATA: {
|
1393
|
-
|
1394
|
-
if (!
|
1448
|
+
// Flag validation: currently allow no flags
|
1449
|
+
if (!AreInitialMetadataFlagsValid(op->flags)) {
|
1395
1450
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1396
1451
|
goto done_with_error;
|
1397
1452
|
}
|
1398
|
-
if (
|
1453
|
+
if (sent_initial_metadata_) {
|
1399
1454
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1400
1455
|
goto done_with_error;
|
1401
1456
|
}
|
@@ -1403,7 +1458,7 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1403
1458
|
// algorithm by setting the initial metadata with key of
|
1404
1459
|
// GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
|
1405
1460
|
// with the compression algorithm mapped from compression level.
|
1406
|
-
|
1461
|
+
// process compression level
|
1407
1462
|
grpc_compression_level effective_compression_level =
|
1408
1463
|
GRPC_COMPRESS_LEVEL_NONE;
|
1409
1464
|
bool level_set = false;
|
@@ -1413,56 +1468,59 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1413
1468
|
level_set = true;
|
1414
1469
|
} else {
|
1415
1470
|
const grpc_compression_options copts =
|
1416
|
-
|
1471
|
+
channel()->compression_options();
|
1417
1472
|
if (copts.default_level.is_set) {
|
1418
1473
|
level_set = true;
|
1419
1474
|
effective_compression_level = copts.default_level.level;
|
1420
1475
|
}
|
1421
1476
|
}
|
1422
1477
|
// Currently, only server side supports compression level setting.
|
1423
|
-
if (level_set && !
|
1478
|
+
if (level_set && !is_client()) {
|
1424
1479
|
const grpc_compression_algorithm calgo =
|
1425
|
-
|
1480
|
+
encodings_accepted_by_peer_.CompressionAlgorithmForLevel(
|
1426
1481
|
effective_compression_level);
|
1427
1482
|
// The following metadata will be checked and removed by the message
|
1428
1483
|
// compression filter. It will be used as the call's compression
|
1429
1484
|
// algorithm.
|
1430
|
-
|
1431
|
-
grpc_core::GrpcInternalEncodingRequest(), calgo);
|
1485
|
+
send_initial_metadata_.Set(GrpcInternalEncodingRequest(), calgo);
|
1432
1486
|
}
|
1433
1487
|
if (op->data.send_initial_metadata.count > INT_MAX) {
|
1434
1488
|
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1435
1489
|
goto done_with_error;
|
1436
1490
|
}
|
1437
1491
|
stream_op->send_initial_metadata = true;
|
1438
|
-
|
1439
|
-
if (!
|
1440
|
-
|
1441
|
-
|
1492
|
+
sent_initial_metadata_ = true;
|
1493
|
+
if (!PrepareApplicationMetadata(op->data.send_initial_metadata.count,
|
1494
|
+
op->data.send_initial_metadata.metadata,
|
1495
|
+
false)) {
|
1442
1496
|
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1443
1497
|
goto done_with_error;
|
1444
1498
|
}
|
1445
1499
|
// Ignore any te metadata key value pairs specified.
|
1446
|
-
|
1447
|
-
|
1448
|
-
if (
|
1449
|
-
|
1450
|
-
|
1451
|
-
|
1500
|
+
send_initial_metadata_.Remove(TeMetadata());
|
1501
|
+
// TODO(ctiller): just make these the same variable?
|
1502
|
+
if (is_client() && send_deadline() != Timestamp::InfFuture()) {
|
1503
|
+
send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
|
1504
|
+
}
|
1505
|
+
if (is_client()) {
|
1506
|
+
send_initial_metadata_.Set(
|
1507
|
+
WaitForReady(),
|
1508
|
+
WaitForReady::ValueType{
|
1509
|
+
(op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
1510
|
+
(op->flags &
|
1511
|
+
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
1452
1512
|
}
|
1453
1513
|
stream_op_payload->send_initial_metadata.send_initial_metadata =
|
1454
|
-
&
|
1455
|
-
|
1456
|
-
op->flags;
|
1457
|
-
if (call->is_client) {
|
1514
|
+
&send_initial_metadata_;
|
1515
|
+
if (is_client()) {
|
1458
1516
|
stream_op_payload->send_initial_metadata.peer_string =
|
1459
|
-
|
1517
|
+
peer_string_atm_ptr();
|
1460
1518
|
}
|
1461
|
-
|
1519
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1462
1520
|
break;
|
1463
1521
|
}
|
1464
1522
|
case GRPC_OP_SEND_MESSAGE: {
|
1465
|
-
if (!
|
1523
|
+
if (!AreWriteFlagsValid(op->flags)) {
|
1466
1524
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1467
1525
|
goto done_with_error;
|
1468
1526
|
}
|
@@ -1470,59 +1528,61 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1470
1528
|
error = GRPC_CALL_ERROR_INVALID_MESSAGE;
|
1471
1529
|
goto done_with_error;
|
1472
1530
|
}
|
1473
|
-
if (
|
1531
|
+
if (sending_message_) {
|
1474
1532
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1475
1533
|
goto done_with_error;
|
1476
1534
|
}
|
1477
1535
|
uint32_t flags = op->flags;
|
1478
|
-
|
1479
|
-
|
1480
|
-
|
1536
|
+
// If the outgoing buffer is already compressed, mark it as so in the
|
1537
|
+
// flags. These will be picked up by the compression filter and further
|
1538
|
+
// (wasteful) attempts at compression skipped.
|
1481
1539
|
if (op->data.send_message.send_message->data.raw.compression >
|
1482
1540
|
GRPC_COMPRESS_NONE) {
|
1483
1541
|
flags |= GRPC_WRITE_INTERNAL_COMPRESS;
|
1484
1542
|
}
|
1485
1543
|
stream_op->send_message = true;
|
1486
|
-
|
1487
|
-
|
1488
|
-
|
1489
|
-
|
1490
|
-
|
1491
|
-
|
1544
|
+
sending_message_ = true;
|
1545
|
+
send_slice_buffer_.Clear();
|
1546
|
+
grpc_slice_buffer_move_into(
|
1547
|
+
&op->data.send_message.send_message->data.raw.slice_buffer,
|
1548
|
+
send_slice_buffer_.c_slice_buffer());
|
1549
|
+
stream_op_payload->send_message.flags = flags;
|
1550
|
+
stream_op_payload->send_message.send_message = &send_slice_buffer_;
|
1551
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1492
1552
|
break;
|
1493
1553
|
}
|
1494
1554
|
case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
|
1495
|
-
|
1555
|
+
// Flag validation: currently allow no flags
|
1496
1556
|
if (op->flags != 0) {
|
1497
1557
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1498
1558
|
goto done_with_error;
|
1499
1559
|
}
|
1500
|
-
if (!
|
1560
|
+
if (!is_client()) {
|
1501
1561
|
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1502
1562
|
goto done_with_error;
|
1503
1563
|
}
|
1504
|
-
if (
|
1564
|
+
if (sent_final_op_) {
|
1505
1565
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1506
1566
|
goto done_with_error;
|
1507
1567
|
}
|
1508
1568
|
stream_op->send_trailing_metadata = true;
|
1509
|
-
|
1569
|
+
sent_final_op_ = true;
|
1510
1570
|
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1511
|
-
&
|
1512
|
-
|
1571
|
+
&send_trailing_metadata_;
|
1572
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1513
1573
|
break;
|
1514
1574
|
}
|
1515
1575
|
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
1516
|
-
|
1576
|
+
// Flag validation: currently allow no flags
|
1517
1577
|
if (op->flags != 0) {
|
1518
1578
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1519
1579
|
goto done_with_error;
|
1520
1580
|
}
|
1521
|
-
if (
|
1581
|
+
if (is_client()) {
|
1522
1582
|
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1523
1583
|
goto done_with_error;
|
1524
1584
|
}
|
1525
|
-
if (
|
1585
|
+
if (sent_final_op_) {
|
1526
1586
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1527
1587
|
goto done_with_error;
|
1528
1588
|
}
|
@@ -1532,280 +1592,2089 @@ static grpc_call_error call_start_batch(grpc_call* call, const grpc_op* ops,
|
|
1532
1592
|
goto done_with_error;
|
1533
1593
|
}
|
1534
1594
|
stream_op->send_trailing_metadata = true;
|
1535
|
-
|
1595
|
+
sent_final_op_ = true;
|
1536
1596
|
|
1537
|
-
if (!
|
1538
|
-
|
1539
|
-
|
1540
|
-
op->data.send_status_from_server.trailing_metadata_count),
|
1541
|
-
op->data.send_status_from_server.trailing_metadata, 1)) {
|
1597
|
+
if (!PrepareApplicationMetadata(
|
1598
|
+
op->data.send_status_from_server.trailing_metadata_count,
|
1599
|
+
op->data.send_status_from_server.trailing_metadata, true)) {
|
1542
1600
|
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1543
1601
|
goto done_with_error;
|
1544
1602
|
}
|
1545
1603
|
|
1546
1604
|
grpc_error_handle status_error =
|
1547
1605
|
op->data.send_status_from_server.status == GRPC_STATUS_OK
|
1548
|
-
?
|
1606
|
+
? absl::OkStatus()
|
1549
1607
|
: grpc_error_set_int(
|
1550
|
-
|
1551
|
-
|
1552
|
-
GRPC_ERROR_INT_GRPC_STATUS,
|
1608
|
+
GRPC_ERROR_CREATE("Server returned error"),
|
1609
|
+
StatusIntProperty::kRpcStatus,
|
1553
1610
|
static_cast<intptr_t>(
|
1554
1611
|
op->data.send_status_from_server.status));
|
1555
1612
|
if (op->data.send_status_from_server.status_details != nullptr) {
|
1556
|
-
|
1557
|
-
|
1558
|
-
|
1613
|
+
send_trailing_metadata_.Set(
|
1614
|
+
GrpcMessageMetadata(),
|
1615
|
+
Slice(grpc_slice_copy(
|
1559
1616
|
*op->data.send_status_from_server.status_details)));
|
1560
|
-
if (status_error
|
1617
|
+
if (!status_error.ok()) {
|
1561
1618
|
status_error = grpc_error_set_str(
|
1562
|
-
status_error,
|
1563
|
-
|
1619
|
+
status_error, StatusStrProperty::kGrpcMessage,
|
1620
|
+
StringViewFromSlice(
|
1564
1621
|
*op->data.send_status_from_server.status_details));
|
1565
1622
|
}
|
1566
1623
|
}
|
1567
1624
|
|
1568
|
-
|
1569
|
-
GRPC_ERROR_UNREF(status_error);
|
1625
|
+
status_error_.set(status_error);
|
1570
1626
|
|
1571
|
-
|
1572
|
-
|
1573
|
-
op->data.send_status_from_server.status);
|
1627
|
+
send_trailing_metadata_.Set(GrpcStatusMetadata(),
|
1628
|
+
op->data.send_status_from_server.status);
|
1574
1629
|
|
1575
1630
|
// Ignore any te metadata key value pairs specified.
|
1576
|
-
|
1631
|
+
send_trailing_metadata_.Remove(TeMetadata());
|
1577
1632
|
stream_op_payload->send_trailing_metadata.send_trailing_metadata =
|
1578
|
-
&
|
1633
|
+
&send_trailing_metadata_;
|
1579
1634
|
stream_op_payload->send_trailing_metadata.sent =
|
1580
|
-
&
|
1581
|
-
|
1635
|
+
&sent_server_trailing_metadata_;
|
1636
|
+
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1582
1637
|
break;
|
1583
1638
|
}
|
1584
1639
|
case GRPC_OP_RECV_INITIAL_METADATA: {
|
1585
|
-
|
1640
|
+
// Flag validation: currently allow no flags
|
1586
1641
|
if (op->flags != 0) {
|
1587
1642
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1588
1643
|
goto done_with_error;
|
1589
1644
|
}
|
1590
|
-
if (
|
1645
|
+
if (received_initial_metadata_) {
|
1591
1646
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1592
1647
|
goto done_with_error;
|
1593
1648
|
}
|
1594
|
-
|
1595
|
-
|
1649
|
+
received_initial_metadata_ = true;
|
1650
|
+
buffered_metadata_[0] =
|
1596
1651
|
op->data.recv_initial_metadata.recv_initial_metadata;
|
1597
|
-
GRPC_CLOSURE_INIT(
|
1598
|
-
|
1599
|
-
|
1652
|
+
GRPC_CLOSURE_INIT(
|
1653
|
+
&receiving_initial_metadata_ready_,
|
1654
|
+
[](void* bctl, grpc_error_handle error) {
|
1655
|
+
static_cast<BatchControl*>(bctl)->ReceivingInitialMetadataReady(
|
1656
|
+
error);
|
1657
|
+
},
|
1658
|
+
bctl, grpc_schedule_on_exec_ctx);
|
1600
1659
|
stream_op->recv_initial_metadata = true;
|
1601
1660
|
stream_op_payload->recv_initial_metadata.recv_initial_metadata =
|
1602
|
-
&
|
1661
|
+
&recv_initial_metadata_;
|
1603
1662
|
stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
|
1604
|
-
&
|
1605
|
-
if (
|
1663
|
+
&receiving_initial_metadata_ready_;
|
1664
|
+
if (is_client()) {
|
1606
1665
|
stream_op_payload->recv_initial_metadata.trailing_metadata_available =
|
1607
|
-
&
|
1666
|
+
&is_trailers_only_;
|
1608
1667
|
} else {
|
1609
1668
|
stream_op_payload->recv_initial_metadata.peer_string =
|
1610
|
-
|
1669
|
+
peer_string_atm_ptr();
|
1611
1670
|
}
|
1612
|
-
|
1671
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
|
1613
1672
|
break;
|
1614
1673
|
}
|
1615
1674
|
case GRPC_OP_RECV_MESSAGE: {
|
1616
|
-
|
1675
|
+
// Flag validation: currently allow no flags
|
1617
1676
|
if (op->flags != 0) {
|
1618
1677
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1619
1678
|
goto done_with_error;
|
1620
1679
|
}
|
1621
|
-
if (
|
1680
|
+
if (receiving_message_) {
|
1622
1681
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1623
1682
|
goto done_with_error;
|
1624
1683
|
}
|
1625
|
-
|
1684
|
+
receiving_message_ = true;
|
1626
1685
|
stream_op->recv_message = true;
|
1627
|
-
|
1628
|
-
|
1686
|
+
receiving_slice_buffer_.reset();
|
1687
|
+
receiving_buffer_ = op->data.recv_message.recv_message;
|
1688
|
+
stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
|
1689
|
+
receiving_stream_flags_ = 0;
|
1690
|
+
stream_op_payload->recv_message.flags = &receiving_stream_flags_;
|
1629
1691
|
stream_op_payload->recv_message.call_failed_before_recv_message =
|
1630
|
-
&
|
1631
|
-
GRPC_CLOSURE_INIT(
|
1632
|
-
|
1633
|
-
|
1692
|
+
&call_failed_before_recv_message_;
|
1693
|
+
GRPC_CLOSURE_INIT(
|
1694
|
+
&receiving_stream_ready_,
|
1695
|
+
[](void* bctlp, grpc_error_handle error) {
|
1696
|
+
auto* bctl = static_cast<BatchControl*>(bctlp);
|
1697
|
+
auto* call = bctl->call_;
|
1698
|
+
// Yields the call combiner before processing the received
|
1699
|
+
// message.
|
1700
|
+
GRPC_CALL_COMBINER_STOP(call->call_combiner(),
|
1701
|
+
"recv_message_ready");
|
1702
|
+
bctl->ReceivingStreamReady(error);
|
1703
|
+
},
|
1704
|
+
bctl, grpc_schedule_on_exec_ctx);
|
1634
1705
|
stream_op_payload->recv_message.recv_message_ready =
|
1635
|
-
&
|
1636
|
-
|
1706
|
+
&receiving_stream_ready_;
|
1707
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
|
1637
1708
|
break;
|
1638
1709
|
}
|
1639
1710
|
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
1640
|
-
|
1711
|
+
// Flag validation: currently allow no flags
|
1641
1712
|
if (op->flags != 0) {
|
1642
1713
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1643
1714
|
goto done_with_error;
|
1644
1715
|
}
|
1645
|
-
if (!
|
1716
|
+
if (!is_client()) {
|
1646
1717
|
error = GRPC_CALL_ERROR_NOT_ON_SERVER;
|
1647
1718
|
goto done_with_error;
|
1648
1719
|
}
|
1649
|
-
if (
|
1720
|
+
if (requested_final_op_) {
|
1650
1721
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1651
1722
|
goto done_with_error;
|
1652
1723
|
}
|
1653
|
-
|
1654
|
-
|
1724
|
+
requested_final_op_ = true;
|
1725
|
+
buffered_metadata_[1] =
|
1655
1726
|
op->data.recv_status_on_client.trailing_metadata;
|
1656
|
-
|
1657
|
-
|
1727
|
+
final_op_.client.status = op->data.recv_status_on_client.status;
|
1728
|
+
final_op_.client.status_details =
|
1658
1729
|
op->data.recv_status_on_client.status_details;
|
1659
|
-
|
1730
|
+
final_op_.client.error_string =
|
1660
1731
|
op->data.recv_status_on_client.error_string;
|
1661
1732
|
stream_op->recv_trailing_metadata = true;
|
1662
1733
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1663
|
-
&
|
1734
|
+
&recv_trailing_metadata_;
|
1664
1735
|
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1665
|
-
&
|
1666
|
-
GRPC_CLOSURE_INIT(
|
1667
|
-
|
1668
|
-
|
1736
|
+
&final_info_.stats.transport_stream_stats;
|
1737
|
+
GRPC_CLOSURE_INIT(
|
1738
|
+
&receiving_trailing_metadata_ready_,
|
1739
|
+
[](void* bctl, grpc_error_handle error) {
|
1740
|
+
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1741
|
+
error);
|
1742
|
+
},
|
1743
|
+
bctl, grpc_schedule_on_exec_ctx);
|
1669
1744
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1670
|
-
&
|
1671
|
-
|
1745
|
+
&receiving_trailing_metadata_ready_;
|
1746
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1672
1747
|
break;
|
1673
1748
|
}
|
1674
1749
|
case GRPC_OP_RECV_CLOSE_ON_SERVER: {
|
1675
|
-
|
1750
|
+
// Flag validation: currently allow no flags
|
1676
1751
|
if (op->flags != 0) {
|
1677
1752
|
error = GRPC_CALL_ERROR_INVALID_FLAGS;
|
1678
1753
|
goto done_with_error;
|
1679
1754
|
}
|
1680
|
-
if (
|
1755
|
+
if (is_client()) {
|
1681
1756
|
error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
1682
1757
|
goto done_with_error;
|
1683
1758
|
}
|
1684
|
-
if (
|
1759
|
+
if (requested_final_op_) {
|
1685
1760
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1686
1761
|
goto done_with_error;
|
1687
1762
|
}
|
1688
|
-
|
1689
|
-
|
1690
|
-
op->data.recv_close_on_server.cancelled;
|
1763
|
+
requested_final_op_ = true;
|
1764
|
+
final_op_.server.cancelled = op->data.recv_close_on_server.cancelled;
|
1691
1765
|
stream_op->recv_trailing_metadata = true;
|
1692
1766
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
|
1693
|
-
&
|
1767
|
+
&recv_trailing_metadata_;
|
1694
1768
|
stream_op_payload->recv_trailing_metadata.collect_stats =
|
1695
|
-
&
|
1696
|
-
GRPC_CLOSURE_INIT(
|
1697
|
-
|
1698
|
-
|
1769
|
+
&final_info_.stats.transport_stream_stats;
|
1770
|
+
GRPC_CLOSURE_INIT(
|
1771
|
+
&receiving_trailing_metadata_ready_,
|
1772
|
+
[](void* bctl, grpc_error_handle error) {
|
1773
|
+
static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
|
1774
|
+
error);
|
1775
|
+
},
|
1776
|
+
bctl, grpc_schedule_on_exec_ctx);
|
1699
1777
|
stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
|
1700
|
-
&
|
1701
|
-
|
1778
|
+
&receiving_trailing_metadata_ready_;
|
1779
|
+
pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
|
1702
1780
|
break;
|
1703
1781
|
}
|
1704
1782
|
}
|
1705
1783
|
}
|
1706
1784
|
|
1707
|
-
|
1785
|
+
InternalRef("completion");
|
1708
1786
|
if (!is_notify_tag_closure) {
|
1709
|
-
GPR_ASSERT(grpc_cq_begin_op(
|
1787
|
+
GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag));
|
1788
|
+
}
|
1789
|
+
bctl->set_pending_ops(pending_ops);
|
1790
|
+
|
1791
|
+
if (pending_ops & PendingOpMask(PendingOp::kSends)) {
|
1792
|
+
GRPC_CLOSURE_INIT(
|
1793
|
+
&bctl->finish_batch_,
|
1794
|
+
[](void* bctl, grpc_error_handle error) {
|
1795
|
+
static_cast<BatchControl*>(bctl)->FinishBatch(error);
|
1796
|
+
},
|
1797
|
+
bctl, grpc_schedule_on_exec_ctx);
|
1798
|
+
stream_op->on_complete = &bctl->finish_batch_;
|
1710
1799
|
}
|
1711
|
-
bctl->set_num_steps_to_complete((has_send_ops ? 1 : 0) + num_recv_ops);
|
1712
1800
|
|
1713
|
-
if (
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1801
|
+
if (grpc_call_trace.enabled()) {
|
1802
|
+
gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1803
|
+
PendingOpString(pending_ops).c_str(),
|
1804
|
+
grpc_transport_stream_op_batch_string(stream_op).c_str(),
|
1805
|
+
bctl->completion_data_.notify_tag.tag);
|
1717
1806
|
}
|
1718
1807
|
|
1719
|
-
|
1720
|
-
execute_batch(call, stream_op, &bctl->start_batch);
|
1808
|
+
ExecuteBatch(stream_op, &bctl->start_batch_);
|
1721
1809
|
|
1722
1810
|
done:
|
1723
1811
|
return error;
|
1724
1812
|
|
1725
1813
|
done_with_error:
|
1726
|
-
|
1814
|
+
// reverse any mutations that occurred
|
1727
1815
|
if (stream_op->send_initial_metadata) {
|
1728
|
-
|
1729
|
-
|
1816
|
+
sent_initial_metadata_ = false;
|
1817
|
+
send_initial_metadata_.Clear();
|
1730
1818
|
}
|
1731
1819
|
if (stream_op->send_message) {
|
1732
|
-
|
1733
|
-
// No need to invoke call->sending_stream->Orphan() explicitly.
|
1734
|
-
// stream_op_payload->send_message.send_message.reset() calls Deletor
|
1735
|
-
// of call->sending_stream which in-turn invokes the Orphan() method.
|
1736
|
-
stream_op_payload->send_message.send_message.reset();
|
1820
|
+
sending_message_ = false;
|
1737
1821
|
}
|
1738
1822
|
if (stream_op->send_trailing_metadata) {
|
1739
|
-
|
1740
|
-
|
1823
|
+
sent_final_op_ = false;
|
1824
|
+
send_trailing_metadata_.Clear();
|
1741
1825
|
}
|
1742
1826
|
if (stream_op->recv_initial_metadata) {
|
1743
|
-
|
1827
|
+
received_initial_metadata_ = false;
|
1744
1828
|
}
|
1745
1829
|
if (stream_op->recv_message) {
|
1746
|
-
|
1830
|
+
receiving_message_ = false;
|
1747
1831
|
}
|
1748
1832
|
if (stream_op->recv_trailing_metadata) {
|
1749
|
-
|
1833
|
+
requested_final_op_ = false;
|
1750
1834
|
}
|
1751
1835
|
goto done;
|
1752
1836
|
}
|
1753
1837
|
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1838
|
+
void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
|
1839
|
+
void (*destroy)(void*)) {
|
1840
|
+
if (context_[elem].destroy) {
|
1841
|
+
context_[elem].destroy(context_[elem].value);
|
1842
|
+
}
|
1843
|
+
context_[elem].value = value;
|
1844
|
+
context_[elem].destroy = destroy;
|
1845
|
+
}
|
1846
|
+
|
1847
|
+
///////////////////////////////////////////////////////////////////////////////
|
1848
|
+
// Metadata validation helpers
|
1849
|
+
|
1850
|
+
namespace {
|
1851
|
+
bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
|
1852
|
+
for (size_t i = 0; i < count; i++) {
|
1853
|
+
grpc_metadata* md = &metadata[i];
|
1854
|
+
if (!GRPC_LOG_IF_ERROR("validate_metadata",
|
1855
|
+
grpc_validate_header_key_is_legal(md->key))) {
|
1856
|
+
return false;
|
1857
|
+
} else if (!grpc_is_binary_header_internal(md->key) &&
|
1858
|
+
!GRPC_LOG_IF_ERROR(
|
1859
|
+
"validate_metadata",
|
1860
|
+
grpc_validate_header_nonbin_value_is_legal(md->value))) {
|
1861
|
+
return false;
|
1862
|
+
} else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
|
1863
|
+
// HTTP2 hpack encoding has a maximum limit.
|
1864
|
+
return false;
|
1865
|
+
}
|
1866
|
+
}
|
1867
|
+
return true;
|
1868
|
+
}
|
1869
|
+
} // namespace
|
1870
|
+
|
1871
|
+
///////////////////////////////////////////////////////////////////////////////
|
1872
|
+
// PromiseBasedCall
|
1873
|
+
// Will be folded into Call once the promise conversion is done
|
1874
|
+
|
1875
|
+
class PromiseBasedCall : public Call,
|
1876
|
+
public Activity,
|
1877
|
+
public Wakeable,
|
1878
|
+
public grpc_event_engine::experimental::EventEngine::
|
1879
|
+
Closure /* for deadlines */ {
|
1880
|
+
public:
|
1881
|
+
PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
1882
|
+
const grpc_call_create_args& args);
|
1883
|
+
|
1884
|
+
void ContextSet(grpc_context_index elem, void* value,
|
1885
|
+
void (*destroy)(void* value)) override;
|
1886
|
+
void* ContextGet(grpc_context_index elem) const override;
|
1887
|
+
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
1888
|
+
void SetCompletionQueueLocked(grpc_completion_queue* cq)
|
1889
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
1890
|
+
void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) {
|
1891
|
+
MutexLock lock(&mu_);
|
1892
|
+
CancelWithErrorLocked(std::move(error));
|
1893
|
+
}
|
1894
|
+
virtual void CancelWithErrorLocked(absl::Status error)
|
1895
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
1896
|
+
bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) {
|
1897
|
+
MutexLock lock(&mu_);
|
1898
|
+
return completed_;
|
1899
|
+
}
|
1900
|
+
|
1901
|
+
void Orphan() final {
|
1902
|
+
MutexLock lock(&mu_);
|
1903
|
+
if (!completed_) {
|
1904
|
+
CancelWithErrorLocked(absl::CancelledError("Call orphaned"));
|
1905
|
+
}
|
1906
|
+
}
|
1907
|
+
|
1908
|
+
// Implementation of call refcounting: move this to DualRefCounted once we
|
1909
|
+
// don't need to maintain FilterStackCall compatibility
|
1910
|
+
void ExternalRef() final {
|
1911
|
+
const uint64_t prev_ref_pair =
|
1912
|
+
refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed);
|
1913
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1914
|
+
gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(),
|
1915
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1916
|
+
GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair));
|
1917
|
+
}
|
1918
|
+
}
|
1919
|
+
void ExternalUnref() final {
|
1920
|
+
const uint64_t prev_ref_pair =
|
1921
|
+
refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel);
|
1922
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1923
|
+
gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(),
|
1924
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1925
|
+
GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1);
|
1926
|
+
}
|
1927
|
+
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
1928
|
+
if (GPR_UNLIKELY(strong_refs == 1)) {
|
1929
|
+
Orphan();
|
1930
|
+
}
|
1931
|
+
// Now drop the weak ref.
|
1932
|
+
InternalUnref("external_ref");
|
1933
|
+
}
|
1934
|
+
void InternalRef(const char* reason) final {
|
1935
|
+
uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed);
|
1936
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1937
|
+
gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason,
|
1938
|
+
GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n),
|
1939
|
+
GetWeakRefs(n) + 1);
|
1940
|
+
}
|
1941
|
+
}
|
1942
|
+
void InternalUnref(const char* reason) final {
|
1943
|
+
const uint64_t prev_ref_pair =
|
1944
|
+
refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel);
|
1945
|
+
if (grpc_call_refcount_trace.enabled()) {
|
1946
|
+
gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(),
|
1947
|
+
reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1948
|
+
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1);
|
1949
|
+
}
|
1950
|
+
if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
|
1951
|
+
DeleteThis();
|
1952
|
+
}
|
1953
|
+
}
|
1954
|
+
|
1955
|
+
// Activity methods
|
1956
|
+
void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1957
|
+
Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override {
|
1958
|
+
InternalRef("wakeup");
|
1959
|
+
// If ASAN is defined, we leverage it to detect dropped Waker objects.
|
1960
|
+
// Usually Waker must be destroyed or woken up, but (especially with arenas)
|
1961
|
+
// it's not uncommon to create a Waker and then do neither. In that case it's
|
1962
|
+
// incredibly fraught to diagnose where the dropped reference to this object was
|
1963
|
+
// created. Instead, leverage ASAN and create a new object per expected wakeup.
|
1964
|
+
// Now when we drop such an object ASAN will fail and we'll get a callstack to
|
1965
|
+
// the creation of the waker in question.
|
1966
|
+
#if defined(__has_feature)
|
1967
|
+
#if __has_feature(address_sanitizer)
|
1968
|
+
#define GRPC_CALL_USES_ASAN_WAKER
|
1969
|
+
class AsanWaker final : public Wakeable {
|
1970
|
+
public:
|
1971
|
+
explicit AsanWaker(PromiseBasedCall* call) : call_(call) {}
|
1972
|
+
|
1973
|
+
void Wakeup(void*) override {
|
1974
|
+
call_->Wakeup(nullptr);
|
1975
|
+
delete this;
|
1976
|
+
}
|
1977
|
+
|
1978
|
+
void Drop(void*) override {
|
1979
|
+
call_->Drop(nullptr);
|
1980
|
+
delete this;
|
1981
|
+
}
|
1982
|
+
|
1983
|
+
std::string ActivityDebugTag(void*) const override {
|
1984
|
+
return call_->DebugTag();
|
1985
|
+
}
|
1986
|
+
|
1987
|
+
private:
|
1988
|
+
PromiseBasedCall* call_;
|
1989
|
+
};
|
1990
|
+
return Waker(new AsanWaker(this), nullptr);
|
1991
|
+
#endif
|
1992
|
+
#endif
|
1993
|
+
#ifndef GRPC_CALL_USES_ASAN_WAKER
|
1994
|
+
return Waker(this, nullptr);
|
1995
|
+
#endif
|
1996
|
+
}
|
1997
|
+
Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1998
|
+
|
1999
|
+
// Wakeable methods
|
2000
|
+
void Wakeup(void*) override {
|
2001
|
+
channel()->event_engine()->Run([this] {
|
2002
|
+
ApplicationCallbackExecCtx app_exec_ctx;
|
2003
|
+
ExecCtx exec_ctx;
|
2004
|
+
{
|
2005
|
+
ScopedContext activity_context(this);
|
2006
|
+
MutexLock lock(&mu_);
|
2007
|
+
Update();
|
2008
|
+
}
|
2009
|
+
InternalUnref("wakeup");
|
2010
|
+
});
|
2011
|
+
}
|
2012
|
+
void Drop(void*) override { InternalUnref("wakeup"); }
|
2013
|
+
|
2014
|
+
void RunInContext(absl::AnyInvocable<void()> fn) {
|
2015
|
+
if (Activity::current() == this) {
|
2016
|
+
fn();
|
2017
|
+
} else {
|
2018
|
+
InternalRef("in_context");
|
2019
|
+
channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable {
|
2020
|
+
ApplicationCallbackExecCtx app_exec_ctx;
|
2021
|
+
ExecCtx exec_ctx;
|
2022
|
+
{
|
2023
|
+
ScopedContext activity_context(this);
|
2024
|
+
MutexLock lock(&mu_);
|
2025
|
+
fn();
|
2026
|
+
Update();
|
2027
|
+
}
|
2028
|
+
InternalUnref("in_context");
|
2029
|
+
});
|
2030
|
+
}
|
2031
|
+
}
|
2032
|
+
|
2033
|
+
grpc_compression_algorithm test_only_compression_algorithm() override {
|
2034
|
+
abort();
|
2035
|
+
}
|
2036
|
+
uint32_t test_only_message_flags() override { abort(); }
|
2037
|
+
uint32_t test_only_encodings_accepted_by_peer() override { abort(); }
|
2038
|
+
grpc_compression_algorithm compression_for_level(
|
2039
|
+
grpc_compression_level) override {
|
2040
|
+
abort();
|
2041
|
+
}
|
2042
|
+
|
2043
|
+
// This should return nullptr for the promise stack (and alternative means
|
2044
|
+
// for that functionality be invented)
|
2045
|
+
grpc_call_stack* call_stack() override { return nullptr; }
|
2046
|
+
|
2047
|
+
void UpdateDeadline(Timestamp deadline);
|
2048
|
+
void ResetDeadline();
|
2049
|
+
|
2050
|
+
// Implementation of EventEngine::Closure, called when deadline expires
|
2051
|
+
void Run() override;
|
2052
|
+
|
2053
|
+
virtual ServerCallContext* server_call_context() { return nullptr; }
|
2054
|
+
|
2055
|
+
protected:
|
2056
|
+
class ScopedContext
|
2057
|
+
: public ScopedActivity,
|
2058
|
+
public promise_detail::Context<Arena>,
|
2059
|
+
public promise_detail::Context<grpc_call_context_element>,
|
2060
|
+
public promise_detail::Context<CallContext>,
|
2061
|
+
public promise_detail::Context<CallFinalization> {
|
2062
|
+
public:
|
2063
|
+
explicit ScopedContext(PromiseBasedCall* call)
|
2064
|
+
: ScopedActivity(call),
|
2065
|
+
promise_detail::Context<Arena>(call->arena()),
|
2066
|
+
promise_detail::Context<grpc_call_context_element>(call->context_),
|
2067
|
+
promise_detail::Context<CallContext>(&call->call_context_),
|
2068
|
+
promise_detail::Context<CallFinalization>(&call->finalization_) {}
|
2069
|
+
};
|
2070
|
+
|
2071
|
+
class Completion {
|
2072
|
+
public:
|
2073
|
+
Completion() : index_(kNullIndex) {}
|
2074
|
+
~Completion() { GPR_ASSERT(index_ == kNullIndex); }
|
2075
|
+
explicit Completion(uint8_t index) : index_(index) {}
|
2076
|
+
Completion(const Completion& other) = delete;
|
2077
|
+
Completion& operator=(const Completion& other) = delete;
|
2078
|
+
Completion(Completion&& other) noexcept : index_(other.index_) {
|
2079
|
+
other.index_ = kNullIndex;
|
2080
|
+
}
|
2081
|
+
Completion& operator=(Completion&& other) noexcept {
|
2082
|
+
GPR_ASSERT(index_ == kNullIndex);
|
2083
|
+
index_ = other.index_;
|
2084
|
+
other.index_ = kNullIndex;
|
2085
|
+
return *this;
|
2086
|
+
}
|
2087
|
+
|
2088
|
+
uint8_t index() const { return index_; }
|
2089
|
+
uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
|
2090
|
+
bool has_value() const { return index_ != kNullIndex; }
|
2091
|
+
|
2092
|
+
private:
|
2093
|
+
enum : uint8_t { kNullIndex = 0xff };
|
2094
|
+
uint8_t index_;
|
2095
|
+
};
|
2096
|
+
|
2097
|
+
~PromiseBasedCall() override {
|
2098
|
+
if (non_owning_wakeable_) non_owning_wakeable_->DropActivity();
|
2099
|
+
if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
|
2100
|
+
}
|
2101
|
+
|
2102
|
+
// Enumerates why a Completion is still pending
|
2103
|
+
enum class PendingOp {
|
2104
|
+
// We're in the midst of starting a batch of operations
|
2105
|
+
kStartingBatch = 0,
|
2106
|
+
// The following correspond with the batch operations from above
|
2107
|
+
kReceiveInitialMetadata,
|
2108
|
+
kReceiveStatusOnClient,
|
2109
|
+
kReceiveCloseOnServer = kReceiveStatusOnClient,
|
2110
|
+
kSendMessage,
|
2111
|
+
kReceiveMessage,
|
2112
|
+
kSendStatusFromServer,
|
2113
|
+
kSendCloseFromClient = kSendStatusFromServer,
|
2114
|
+
};
|
2115
|
+
|
2116
|
+
const char* PendingOpString(PendingOp reason) const {
|
2117
|
+
switch (reason) {
|
2118
|
+
case PendingOp::kStartingBatch:
|
2119
|
+
return "StartingBatch";
|
2120
|
+
case PendingOp::kReceiveInitialMetadata:
|
2121
|
+
return "ReceiveInitialMetadata";
|
2122
|
+
case PendingOp::kReceiveStatusOnClient:
|
2123
|
+
return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
|
2124
|
+
case PendingOp::kSendMessage:
|
2125
|
+
return "SendMessage";
|
2126
|
+
case PendingOp::kReceiveMessage:
|
2127
|
+
return "ReceiveMessage";
|
2128
|
+
case PendingOp::kSendStatusFromServer:
|
2129
|
+
return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
|
2130
|
+
}
|
2131
|
+
return "Unknown";
|
2132
|
+
}
|
2133
|
+
|
2134
|
+
static constexpr uint8_t PendingOpBit(PendingOp reason) {
|
2135
|
+
return 1 << static_cast<int>(reason);
|
2136
|
+
}
|
2137
|
+
|
2138
|
+
Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; }
|
2139
|
+
// Begin work on a completion, recording the tag/closure to notify.
|
2140
|
+
// Use the op selected in \a ops to determine the index to allocate into.
|
2141
|
+
// Starts the "StartingBatch" PendingOp immediately.
|
2142
|
+
// Assumes at least one operation in \a ops.
|
2143
|
+
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops)
|
2144
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2145
|
+
// Add one pending op to the completion, and return it.
|
2146
|
+
Completion AddOpToCompletion(const Completion& completion, PendingOp reason)
|
2147
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2148
|
+
// Stringify a completion
|
2149
|
+
std::string CompletionString(const Completion& completion) const {
|
2150
|
+
return completion.has_value()
|
2151
|
+
? absl::StrFormat(
|
2152
|
+
"%d:tag=%p", static_cast<int>(completion.index()),
|
2153
|
+
completion_info_[completion.index()].pending.tag)
|
2154
|
+
: "no-completion";
|
2155
|
+
}
|
2156
|
+
// Finish one op on the completion. Must have been previously been added.
|
2157
|
+
// The completion as a whole finishes when all pending ops finish.
|
2158
|
+
void FinishOpOnCompletion(Completion* completion, PendingOp reason)
|
2159
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2160
|
+
// Mark the completion as failed. Does not finish it.
|
2161
|
+
void FailCompletion(const Completion& completion,
|
2162
|
+
SourceLocation source_location = {});
|
2163
|
+
// Run the promise polling loop until it stalls.
|
2164
|
+
void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2165
|
+
// Update the promise state once.
|
2166
|
+
virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
2167
|
+
// Accept the stats from the context (call once we have proof the transport is
|
2168
|
+
// done with them).
|
2169
|
+
// Right now this means that promise based calls do not record correct stats
|
2170
|
+
// with census if they are cancelled.
|
2171
|
+
// TODO(ctiller): this should be remedied before promise based calls are
|
2172
|
+
// dexperimentalized.
|
2173
|
+
void AcceptTransportStatsFromContext() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2174
|
+
final_stats_ = *call_context_.call_stats();
|
2175
|
+
}
|
2176
|
+
|
2177
|
+
grpc_completion_queue* cq() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return cq_; }
|
2178
|
+
|
2179
|
+
void CToMetadata(grpc_metadata* metadata, size_t count,
|
2180
|
+
grpc_metadata_batch* batch);
|
2181
|
+
|
2182
|
+
std::string ActivityDebugTag(void*) const override { return DebugTag(); }
|
2183
|
+
|
2184
|
+
// At the end of the call run any finalization actions.
|
2185
|
+
void RunFinalization(grpc_status_code status, const char* status_details) {
|
2186
|
+
grpc_call_final_info final_info;
|
2187
|
+
final_info.stats = final_stats_;
|
2188
|
+
final_info.final_status = status;
|
2189
|
+
final_info.error_string = status_details;
|
2190
|
+
finalization_.Run(&final_info);
|
2191
|
+
}
|
2192
|
+
|
2193
|
+
std::string PresentAndCompletionText(const char* caption, bool has,
|
2194
|
+
const Completion& completion) const {
|
2195
|
+
if (has) {
|
2196
|
+
if (completion.has_value()) {
|
2197
|
+
return absl::StrCat(caption, ":", CompletionString(completion), " ");
|
2198
|
+
} else {
|
2199
|
+
return absl::StrCat(caption,
|
2200
|
+
":!!BUG:operation is present, no completion!! ");
|
2201
|
+
}
|
2202
|
+
} else {
|
2203
|
+
if (!completion.has_value()) {
|
2204
|
+
return "";
|
2205
|
+
} else {
|
2206
|
+
return absl::StrCat(caption, ":no-op:", CompletionString(completion),
|
2207
|
+
" ");
|
2208
|
+
}
|
2209
|
+
}
|
2210
|
+
}
|
2211
|
+
|
2212
|
+
std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2213
|
+
return absl::StrCat(PresentAndCompletionText("outstanding_send",
|
2214
|
+
outstanding_send_.has_value(),
|
2215
|
+
send_message_completion_)
|
2216
|
+
.c_str(),
|
2217
|
+
PresentAndCompletionText("outstanding_recv",
|
2218
|
+
outstanding_recv_.has_value(),
|
2219
|
+
recv_message_completion_)
|
2220
|
+
.c_str());
|
2221
|
+
}
|
2222
|
+
|
2223
|
+
void StartRecvMessage(const grpc_op& op, const Completion& completion,
|
2224
|
+
PipeReceiver<MessageHandle>* receiver)
|
2225
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2226
|
+
void PollRecvMessage(grpc_compression_algorithm compression_algorithm)
|
2227
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2228
|
+
void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2229
|
+
void StartSendMessage(const grpc_op& op, const Completion& completion,
|
2230
|
+
PipeSender<MessageHandle>* sender)
|
2231
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2232
|
+
bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2233
|
+
void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2234
|
+
|
2235
|
+
bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2236
|
+
return completed_;
|
2237
|
+
}
|
2238
|
+
void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; }
|
2239
|
+
bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
2240
|
+
return outstanding_send_.has_value();
|
2241
|
+
}
|
2242
|
+
|
2243
|
+
private:
|
2244
|
+
union CompletionInfo {
|
2245
|
+
struct Pending {
|
2246
|
+
// Bitmask of PendingOps
|
2247
|
+
uint8_t pending_op_bits;
|
2248
|
+
bool is_closure;
|
2249
|
+
bool success;
|
2250
|
+
void* tag;
|
2251
|
+
} pending;
|
2252
|
+
grpc_cq_completion completion;
|
2253
|
+
};
|
2254
|
+
|
2255
|
+
class NonOwningWakable final : public Wakeable {
|
2256
|
+
public:
|
2257
|
+
explicit NonOwningWakable(PromiseBasedCall* call) : call_(call) {}
|
2258
|
+
|
2259
|
+
// Ref the Handle (not the activity).
|
2260
|
+
void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
|
2261
|
+
|
2262
|
+
// Activity is going away... drop its reference and sever the connection
|
2263
|
+
// back.
|
2264
|
+
void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
|
2265
|
+
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2266
|
+
MutexLock lock(&mu_);
|
2267
|
+
GPR_ASSERT(call_ != nullptr);
|
2268
|
+
call_ = nullptr;
|
2269
|
+
}
|
2270
|
+
|
2271
|
+
// Activity needs to wake up (if it still exists!) - wake it up, and drop
|
2272
|
+
// the ref that was kept for this handle.
|
2273
|
+
void Wakeup(void*) override ABSL_LOCKS_EXCLUDED(mu_) {
|
2274
|
+
// Drop the ref to the handle at end of scope (we have one ref = one
|
2275
|
+
// wakeup semantics).
|
2276
|
+
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2277
|
+
ReleasableMutexLock lock(&mu_);
|
2278
|
+
// Note that activity refcount can drop to zero, but we could win the lock
|
2279
|
+
// against DropActivity, so we need to only increase activities refcount
|
2280
|
+
// if it is non-zero.
|
2281
|
+
PromiseBasedCall* call = call_;
|
2282
|
+
if (call != nullptr && call->RefIfNonZero()) {
|
2283
|
+
lock.Release();
|
2284
|
+
// Activity still exists and we have a reference: wake it up, which will
|
2285
|
+
// drop the ref.
|
2286
|
+
call->Wakeup(nullptr);
|
2287
|
+
}
|
2288
|
+
}
|
2289
|
+
|
2290
|
+
std::string ActivityDebugTag(void*) const override {
|
2291
|
+
MutexLock lock(&mu_);
|
2292
|
+
return call_ == nullptr ? "<unknown>" : call_->DebugTag();
|
2293
|
+
}
|
2294
|
+
|
2295
|
+
void Drop(void*) override { Unref(); }
|
2296
|
+
|
2297
|
+
private:
|
2298
|
+
// Unref the Handle (not the activity).
|
2299
|
+
void Unref() {
|
2300
|
+
if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) {
|
2301
|
+
delete this;
|
2302
|
+
}
|
2303
|
+
}
|
2304
|
+
|
2305
|
+
mutable Mutex mu_;
|
2306
|
+
// We have two initial refs: one for the wakeup that this is created for,
|
2307
|
+
// and will be dropped by Wakeup, and the other for the activity which is
|
2308
|
+
// dropped by DropActivity.
|
2309
|
+
std::atomic<size_t> refs_{2};
|
2310
|
+
PromiseBasedCall* call_ ABSL_GUARDED_BY(mu_);
|
2311
|
+
};
|
2312
|
+
|
2313
|
+
static void OnDestroy(void* arg, grpc_error_handle) {
|
2314
|
+
auto* call = static_cast<PromiseBasedCall*>(arg);
|
2315
|
+
ScopedContext context(call);
|
2316
|
+
call->DeleteThis();
|
2317
|
+
}
|
2318
|
+
|
2319
|
+
// First 32 bits are strong refs, next 32 bits are weak refs.
|
2320
|
+
static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
|
2321
|
+
return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
|
2322
|
+
}
|
2323
|
+
static uint32_t GetStrongRefs(uint64_t ref_pair) {
|
2324
|
+
return static_cast<uint32_t>(ref_pair >> 32);
|
2325
|
+
}
|
2326
|
+
static uint32_t GetWeakRefs(uint64_t ref_pair) {
|
2327
|
+
return static_cast<uint32_t>(ref_pair & 0xffffffffu);
|
2328
|
+
}
|
2329
|
+
|
2330
|
+
bool RefIfNonZero() {
|
2331
|
+
uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire);
|
2332
|
+
do {
|
2333
|
+
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
2334
|
+
if (strong_refs == 0) return false;
|
2335
|
+
} while (!refs_.compare_exchange_weak(
|
2336
|
+
prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0),
|
2337
|
+
std::memory_order_acq_rel, std::memory_order_acquire));
|
2338
|
+
return true;
|
2339
|
+
}
|
2340
|
+
|
2341
|
+
mutable Mutex mu_;
|
2342
|
+
std::atomic<uint64_t> refs_;
|
2343
|
+
CallContext call_context_{this};
|
2344
|
+
bool keep_polling_ ABSL_GUARDED_BY(mu()) = false;
|
2345
|
+
|
2346
|
+
// Contexts for various subsystems (security, tracing, ...).
|
2347
|
+
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
2348
|
+
grpc_completion_queue* cq_ ABSL_GUARDED_BY(mu_);
|
2349
|
+
NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr;
|
2350
|
+
CompletionInfo completion_info_[6];
|
2351
|
+
grpc_call_stats final_stats_{};
|
2352
|
+
CallFinalization finalization_;
|
2353
|
+
// Current deadline.
|
2354
|
+
Timestamp deadline_ = Timestamp::InfFuture();
|
2355
|
+
grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_;
|
2356
|
+
absl::optional<PipeSender<MessageHandle>::PushType> outstanding_send_
|
2357
|
+
ABSL_GUARDED_BY(mu_);
|
2358
|
+
absl::optional<PipeReceiverNextType<MessageHandle>> outstanding_recv_
|
2359
|
+
ABSL_GUARDED_BY(mu_);
|
2360
|
+
grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr;
|
2361
|
+
Completion send_message_completion_ ABSL_GUARDED_BY(mu_);
|
2362
|
+
Completion recv_message_completion_ ABSL_GUARDED_BY(mu_);
|
2363
|
+
bool completed_ ABSL_GUARDED_BY(mu_) = false;
|
2364
|
+
};
|
2365
|
+
|
2366
|
+
template <typename T>
|
2367
|
+
grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
|
2368
|
+
grpc_call** out_call) {
|
2369
|
+
Channel* channel = args->channel.get();
|
2370
|
+
|
2371
|
+
auto alloc = Arena::CreateWithAlloc(channel->CallSizeEstimate(), sizeof(T),
|
2372
|
+
channel->allocator());
|
2373
|
+
PromiseBasedCall* call = new (alloc.second) T(alloc.first, args);
|
2374
|
+
*out_call = call->c_ptr();
|
2375
|
+
GPR_DEBUG_ASSERT(Call::FromC(*out_call) == call);
|
2376
|
+
return absl::OkStatus();
|
2377
|
+
}
|
2378
|
+
|
2379
|
+
PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
2380
|
+
const grpc_call_create_args& args)
|
2381
|
+
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
2382
|
+
args.channel->Ref()),
|
2383
|
+
refs_(MakeRefPair(initial_external_refs, 0)),
|
2384
|
+
cq_(args.cq) {
|
2385
|
+
if (args.cq != nullptr) {
|
2386
|
+
GPR_ASSERT(args.pollset_set_alternative == nullptr &&
|
2387
|
+
"Only one of 'cq' and 'pollset_set_alternative' should be "
|
2388
|
+
"non-nullptr.");
|
2389
|
+
GRPC_CQ_INTERNAL_REF(args.cq, "bind");
|
2390
|
+
call_context_.pollent_ =
|
2391
|
+
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args.cq));
|
2392
|
+
}
|
2393
|
+
if (args.pollset_set_alternative != nullptr) {
|
2394
|
+
call_context_.pollent_ = grpc_polling_entity_create_from_pollset_set(
|
2395
|
+
args.pollset_set_alternative);
|
2396
|
+
}
|
2397
|
+
}
|
2398
|
+
|
2399
|
+
Waker PromiseBasedCall::MakeNonOwningWaker() {
|
2400
|
+
if (non_owning_wakeable_ == nullptr) {
|
2401
|
+
non_owning_wakeable_ = new NonOwningWakable(this);
|
2402
|
+
} else {
|
2403
|
+
non_owning_wakeable_->Ref();
|
2404
|
+
}
|
2405
|
+
return Waker(non_owning_wakeable_, nullptr);
|
2406
|
+
}
|
2407
|
+
|
2408
|
+
void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count,
|
2409
|
+
grpc_metadata_batch* b) {
|
2410
|
+
for (size_t i = 0; i < count; i++) {
|
2411
|
+
grpc_metadata* md = &metadata[i];
|
2412
|
+
auto key = StringViewFromSlice(md->key);
|
2413
|
+
// Filter "content-length metadata"
|
2414
|
+
if (key == "content-length") continue;
|
2415
|
+
b->Append(key, Slice(CSliceRef(md->value)),
|
2416
|
+
[md](absl::string_view error, const Slice& value) {
|
2417
|
+
gpr_log(GPR_DEBUG, "Append error: %s",
|
2418
|
+
absl::StrCat("key=", StringViewFromSlice(md->key),
|
2419
|
+
" error=", error,
|
2420
|
+
" value=", value.as_string_view())
|
2421
|
+
.c_str());
|
2422
|
+
});
|
2423
|
+
}
|
2424
|
+
}
|
1757
2425
|
|
2426
|
+
void PromiseBasedCall::ContextSet(grpc_context_index elem, void* value,
|
2427
|
+
void (*destroy)(void*)) {
|
2428
|
+
if (context_[elem].destroy != nullptr) {
|
2429
|
+
context_[elem].destroy(context_[elem].value);
|
2430
|
+
}
|
2431
|
+
context_[elem].value = value;
|
2432
|
+
context_[elem].destroy = destroy;
|
2433
|
+
}
|
2434
|
+
|
2435
|
+
void* PromiseBasedCall::ContextGet(grpc_context_index elem) const {
|
2436
|
+
return context_[elem].value;
|
2437
|
+
}
|
2438
|
+
|
2439
|
+
PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
|
2440
|
+
void* tag, bool is_closure, const grpc_op* ops) {
|
2441
|
+
Completion c(BatchSlotForOp(ops[0].op));
|
2442
|
+
if (grpc_call_trace.enabled()) {
|
2443
|
+
gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(),
|
2444
|
+
CompletionString(c).c_str(), tag);
|
2445
|
+
}
|
2446
|
+
if (!is_closure) {
|
2447
|
+
grpc_cq_begin_op(cq(), tag);
|
2448
|
+
}
|
2449
|
+
completion_info_[c.index()].pending = {
|
2450
|
+
PendingOpBit(PendingOp::kStartingBatch), is_closure, true, tag};
|
2451
|
+
return c;
|
2452
|
+
}
|
2453
|
+
|
2454
|
+
PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
|
2455
|
+
const Completion& completion, PendingOp reason) {
|
2456
|
+
if (grpc_call_trace.enabled()) {
|
2457
|
+
gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
|
2458
|
+
CompletionString(completion).c_str(), PendingOpString(reason));
|
2459
|
+
}
|
2460
|
+
GPR_ASSERT(completion.has_value());
|
2461
|
+
auto& pending_op_bits =
|
2462
|
+
completion_info_[completion.index()].pending.pending_op_bits;
|
2463
|
+
GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0);
|
2464
|
+
pending_op_bits |= PendingOpBit(reason);
|
2465
|
+
return Completion(completion.index());
|
2466
|
+
}
|
2467
|
+
|
2468
|
+
void PromiseBasedCall::FailCompletion(const Completion& completion,
|
2469
|
+
SourceLocation location) {
|
2470
|
+
if (grpc_call_trace.enabled()) {
|
2471
|
+
gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
|
2472
|
+
"%s[call] FailCompletion %s", DebugTag().c_str(),
|
2473
|
+
CompletionString(completion).c_str());
|
2474
|
+
}
|
2475
|
+
completion_info_[completion.index()].pending.success = false;
|
2476
|
+
}
|
2477
|
+
|
2478
|
+
void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
|
2479
|
+
PendingOp reason) {
|
2480
|
+
if (grpc_call_trace.enabled()) {
|
2481
|
+
auto pending_op_bits =
|
2482
|
+
completion_info_[completion->index()].pending.pending_op_bits;
|
2483
|
+
bool success = completion_info_[completion->index()].pending.success;
|
2484
|
+
std::vector<const char*> pending;
|
2485
|
+
for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) {
|
2486
|
+
if (static_cast<PendingOp>(i) == reason) continue;
|
2487
|
+
if (pending_op_bits & (1 << i)) {
|
2488
|
+
pending.push_back(PendingOpString(static_cast<PendingOp>(i)));
|
2489
|
+
}
|
2490
|
+
}
|
2491
|
+
gpr_log(
|
2492
|
+
GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s",
|
2493
|
+
DebugTag().c_str(), completion_info_[completion->index()].pending.tag,
|
2494
|
+
CompletionString(*completion).c_str(), PendingOpString(reason),
|
2495
|
+
(pending.empty()
|
2496
|
+
? (success ? std::string("done") : std::string("failed"))
|
2497
|
+
: absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ",")))
|
2498
|
+
.c_str());
|
2499
|
+
}
|
2500
|
+
const uint8_t i = completion->TakeIndex();
|
2501
|
+
GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_));
|
2502
|
+
CompletionInfo::Pending& pending = completion_info_[i].pending;
|
2503
|
+
GPR_ASSERT(pending.pending_op_bits & PendingOpBit(reason));
|
2504
|
+
pending.pending_op_bits &= ~PendingOpBit(reason);
|
2505
|
+
auto error = pending.success ? absl::OkStatus() : absl::CancelledError();
|
2506
|
+
if (pending.pending_op_bits == 0) {
|
2507
|
+
if (pending.is_closure) {
|
2508
|
+
ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
|
2509
|
+
error);
|
2510
|
+
} else {
|
2511
|
+
grpc_cq_end_op(
|
2512
|
+
cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
|
2513
|
+
&completion_info_[i].completion);
|
2514
|
+
}
|
2515
|
+
}
|
2516
|
+
}
|
2517
|
+
|
2518
|
+
void PromiseBasedCall::Update() {
|
2519
|
+
keep_polling_ = false;
|
2520
|
+
do {
|
2521
|
+
UpdateOnce();
|
2522
|
+
} while (std::exchange(keep_polling_, false));
|
2523
|
+
}
|
2524
|
+
|
2525
|
+
void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; }
|
2526
|
+
|
2527
|
+
void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
2528
|
+
MutexLock lock(&mu_);
|
2529
|
+
SetCompletionQueueLocked(cq);
|
2530
|
+
}
|
2531
|
+
|
2532
|
+
void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
|
2533
|
+
cq_ = cq;
|
2534
|
+
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
2535
|
+
call_context_.pollent_ =
|
2536
|
+
grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
|
2537
|
+
}
|
2538
|
+
|
2539
|
+
void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
|
2540
|
+
if (deadline >= deadline_) return;
|
2541
|
+
auto* const event_engine = channel()->event_engine();
|
2542
|
+
if (deadline_ != Timestamp::InfFuture()) {
|
2543
|
+
if (!event_engine->Cancel(deadline_task_)) return;
|
2544
|
+
} else {
|
2545
|
+
InternalRef("deadline");
|
2546
|
+
}
|
2547
|
+
event_engine->RunAfter(deadline - Timestamp::Now(), this);
|
2548
|
+
}
|
2549
|
+
|
2550
|
+
void PromiseBasedCall::ResetDeadline() {
|
2551
|
+
if (deadline_ == Timestamp::InfFuture()) return;
|
2552
|
+
auto* const event_engine = channel()->event_engine();
|
2553
|
+
if (!event_engine->Cancel(deadline_task_)) return;
|
2554
|
+
deadline_ = Timestamp::InfFuture();
|
2555
|
+
InternalUnref("deadline");
|
2556
|
+
}
|
2557
|
+
|
2558
|
+
void PromiseBasedCall::Run() {
|
2559
|
+
ApplicationCallbackExecCtx callback_exec_ctx;
|
2560
|
+
ExecCtx exec_ctx;
|
2561
|
+
CancelWithError(absl::DeadlineExceededError("Deadline exceeded"));
|
2562
|
+
InternalUnref("deadline");
|
2563
|
+
}
|
2564
|
+
|
2565
|
+
void PromiseBasedCall::StartSendMessage(const grpc_op& op,
|
2566
|
+
const Completion& completion,
|
2567
|
+
PipeSender<MessageHandle>* sender) {
|
2568
|
+
GPR_ASSERT(!outstanding_send_.has_value());
|
2569
|
+
if (!completed_) {
|
2570
|
+
send_message_completion_ =
|
2571
|
+
AddOpToCompletion(completion, PendingOp::kSendMessage);
|
2572
|
+
SliceBuffer send;
|
2573
|
+
grpc_slice_buffer_swap(
|
2574
|
+
&op.data.send_message.send_message->data.raw.slice_buffer,
|
2575
|
+
send.c_slice_buffer());
|
2576
|
+
outstanding_send_.emplace(sender->Push(
|
2577
|
+
GetContext<Arena>()->MakePooled<Message>(std::move(send), op.flags)));
|
2578
|
+
} else {
|
2579
|
+
FailCompletion(completion);
|
2580
|
+
}
|
2581
|
+
}
|
2582
|
+
|
2583
|
+
bool PromiseBasedCall::PollSendMessage() {
|
2584
|
+
if (!outstanding_send_.has_value()) return true;
|
2585
|
+
Poll<bool> r = (*outstanding_send_)();
|
2586
|
+
if (const bool* result = r.value_if_ready()) {
|
2587
|
+
if (grpc_call_trace.enabled()) {
|
2588
|
+
gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(),
|
2589
|
+
*result ? "successfully" : "with failure");
|
2590
|
+
}
|
2591
|
+
if (!*result) {
|
2592
|
+
FailCompletion(send_message_completion_);
|
2593
|
+
return false;
|
2594
|
+
}
|
2595
|
+
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2596
|
+
outstanding_send_.reset();
|
2597
|
+
}
|
2598
|
+
return true;
|
2599
|
+
}
|
2600
|
+
|
2601
|
+
void PromiseBasedCall::CancelSendMessage() {
|
2602
|
+
if (!outstanding_send_.has_value()) return;
|
2603
|
+
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2604
|
+
outstanding_send_.reset();
|
2605
|
+
}
|
2606
|
+
|
2607
|
+
void PromiseBasedCall::StartRecvMessage(const grpc_op& op,
|
2608
|
+
const Completion& completion,
|
2609
|
+
PipeReceiver<MessageHandle>* receiver) {
|
2610
|
+
GPR_ASSERT(!outstanding_recv_.has_value());
|
2611
|
+
recv_message_ = op.data.recv_message.recv_message;
|
2612
|
+
recv_message_completion_ =
|
2613
|
+
AddOpToCompletion(completion, PendingOp::kReceiveMessage);
|
2614
|
+
outstanding_recv_.emplace(receiver->Next());
|
2615
|
+
}
|
2616
|
+
|
2617
|
+
void PromiseBasedCall::PollRecvMessage(
|
2618
|
+
grpc_compression_algorithm incoming_compression_algorithm) {
|
2619
|
+
if (!outstanding_recv_.has_value()) return;
|
2620
|
+
Poll<NextResult<MessageHandle>> r = (*outstanding_recv_)();
|
2621
|
+
if (auto* result = r.value_if_ready()) {
|
2622
|
+
outstanding_recv_.reset();
|
2623
|
+
if (result->has_value()) {
|
2624
|
+
MessageHandle& message = **result;
|
2625
|
+
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
2626
|
+
(incoming_compression_algorithm != GRPC_COMPRESS_NONE)) {
|
2627
|
+
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
2628
|
+
nullptr, 0, incoming_compression_algorithm);
|
2629
|
+
} else {
|
2630
|
+
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
2631
|
+
}
|
2632
|
+
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
2633
|
+
&(*recv_message_)->data.raw.slice_buffer);
|
2634
|
+
if (grpc_call_trace.enabled()) {
|
2635
|
+
gpr_log(GPR_INFO,
|
2636
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2637
|
+
"%" PRIdPTR " byte message",
|
2638
|
+
DebugTag().c_str(),
|
2639
|
+
(*recv_message_)->data.raw.slice_buffer.length);
|
2640
|
+
}
|
2641
|
+
} else if (result->cancelled()) {
|
2642
|
+
if (grpc_call_trace.enabled()) {
|
2643
|
+
gpr_log(GPR_INFO,
|
2644
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2645
|
+
"end-of-stream with error",
|
2646
|
+
DebugTag().c_str());
|
2647
|
+
}
|
2648
|
+
FailCompletion(recv_message_completion_);
|
2649
|
+
*recv_message_ = nullptr;
|
2650
|
+
} else {
|
2651
|
+
if (grpc_call_trace.enabled()) {
|
2652
|
+
gpr_log(GPR_INFO,
|
2653
|
+
"%s[call] PollRecvMessage: outstanding_recv finishes: received "
|
2654
|
+
"end-of-stream",
|
2655
|
+
DebugTag().c_str());
|
2656
|
+
}
|
2657
|
+
*recv_message_ = nullptr;
|
2658
|
+
}
|
2659
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2660
|
+
} else if (completed_) {
|
2661
|
+
if (grpc_call_trace.enabled()) {
|
2662
|
+
gpr_log(GPR_INFO,
|
2663
|
+
"%s[call] UpdateOnce: outstanding_recv finishes: promise has "
|
2664
|
+
"completed without queuing a message, forcing end-of-stream",
|
2665
|
+
DebugTag().c_str());
|
2666
|
+
}
|
2667
|
+
outstanding_recv_.reset();
|
2668
|
+
*recv_message_ = nullptr;
|
2669
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2670
|
+
}
|
2671
|
+
}
|
2672
|
+
|
2673
|
+
void PromiseBasedCall::CancelRecvMessage() {
|
2674
|
+
if (!outstanding_recv_.has_value()) return;
|
2675
|
+
*recv_message_ = nullptr;
|
2676
|
+
outstanding_recv_.reset();
|
2677
|
+
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2678
|
+
}
|
2679
|
+
|
2680
|
+
///////////////////////////////////////////////////////////////////////////////
|
2681
|
+
// CallContext
|
2682
|
+
|
2683
|
+
void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
|
2684
|
+
call_->RunInContext(std::move(fn));
|
2685
|
+
}
|
2686
|
+
|
2687
|
+
void CallContext::IncrementRefCount(const char* reason) {
|
2688
|
+
call_->InternalRef(reason);
|
2689
|
+
}
|
2690
|
+
|
2691
|
+
void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
|
2692
|
+
|
2693
|
+
gpr_atm* CallContext::peer_string_atm_ptr() {
|
2694
|
+
return call_->peer_string_atm_ptr();
|
2695
|
+
}
|
2696
|
+
|
2697
|
+
void CallContext::UpdateDeadline(Timestamp deadline) {
|
2698
|
+
call_->UpdateDeadline(deadline);
|
2699
|
+
}
|
2700
|
+
|
2701
|
+
ServerCallContext* CallContext::server_call_context() {
|
2702
|
+
return call_->server_call_context();
|
2703
|
+
}
|
2704
|
+
|
2705
|
+
///////////////////////////////////////////////////////////////////////////////
|
2706
|
+
// PublishMetadataArray
|
2707
|
+
|
2708
|
+
namespace {
|
2709
|
+
void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) {
|
2710
|
+
const auto md_count = md->count();
|
2711
|
+
if (md_count > array->capacity) {
|
2712
|
+
array->capacity =
|
2713
|
+
std::max(array->capacity + md->count(), array->capacity * 3 / 2);
|
2714
|
+
array->metadata = static_cast<grpc_metadata*>(
|
2715
|
+
gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
|
2716
|
+
}
|
2717
|
+
PublishToAppEncoder encoder(array);
|
2718
|
+
md->Encode(&encoder);
|
2719
|
+
}
|
2720
|
+
} // namespace
|
2721
|
+
|
2722
|
+
///////////////////////////////////////////////////////////////////////////////
|
2723
|
+
// ClientPromiseBasedCall
|
2724
|
+
|
2725
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
2726
|
+
class ClientPromiseBasedCall final : public PromiseBasedCall {
|
2727
|
+
public:
|
2728
|
+
ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
|
2729
|
+
: PromiseBasedCall(arena, 1, *args) {
|
2730
|
+
global_stats().IncrementClientCallsCreated();
|
2731
|
+
ScopedContext context(this);
|
2732
|
+
send_initial_metadata_ =
|
2733
|
+
GetContext<Arena>()->MakePooled<ClientMetadata>(GetContext<Arena>());
|
2734
|
+
send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
|
2735
|
+
if (args->authority.has_value()) {
|
2736
|
+
send_initial_metadata_->Set(HttpAuthorityMetadata(),
|
2737
|
+
std::move(*args->authority));
|
2738
|
+
}
|
2739
|
+
if (auto* channelz_channel = channel()->channelz_node()) {
|
2740
|
+
channelz_channel->RecordCallStarted();
|
2741
|
+
}
|
2742
|
+
if (args->send_deadline != Timestamp::InfFuture()) {
|
2743
|
+
UpdateDeadline(args->send_deadline);
|
2744
|
+
}
|
2745
|
+
}
|
2746
|
+
|
2747
|
+
~ClientPromiseBasedCall() override {
|
2748
|
+
ScopedContext context(this);
|
2749
|
+
send_initial_metadata_.reset();
|
2750
|
+
recv_status_on_client_ = absl::monostate();
|
2751
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
2752
|
+
// Need to destroy the pipes under the ScopedContext above, so we move them
|
2753
|
+
// out here and then allow the destructors to run at end of scope, but
|
2754
|
+
// before context.
|
2755
|
+
auto c2s = std::move(client_to_server_messages_);
|
2756
|
+
auto s2c = std::move(server_to_client_messages_);
|
2757
|
+
auto sim = std::move(server_initial_metadata_);
|
2758
|
+
}
|
2759
|
+
|
2760
|
+
absl::string_view GetServerAuthority() const override { abort(); }
|
2761
|
+
void CancelWithErrorLocked(grpc_error_handle error) override
|
2762
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2763
|
+
bool is_trailers_only() const override {
|
2764
|
+
MutexLock lock(mu());
|
2765
|
+
return is_trailers_only_;
|
2766
|
+
}
|
2767
|
+
bool failed_before_recv_message() const override { abort(); }
|
2768
|
+
|
2769
|
+
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
2770
|
+
bool is_notify_tag_closure) override;
|
2771
|
+
|
2772
|
+
std::string DebugTag() const override {
|
2773
|
+
return absl::StrFormat("CLIENT_CALL[%p]: ", this);
|
2774
|
+
}
|
2775
|
+
|
2776
|
+
private:
|
2777
|
+
// Poll the underlying promise (and sundry objects) once.
|
2778
|
+
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
2779
|
+
// Finish the call with the given status/trailing metadata.
|
2780
|
+
void Finish(ServerMetadataHandle trailing_metadata)
|
2781
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2782
|
+
// Validate that a set of ops is valid for a client call.
|
2783
|
+
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const
|
2784
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2785
|
+
// Commit a valid batch of operations to be executed.
|
2786
|
+
void CommitBatch(const grpc_op* ops, size_t nops,
|
2787
|
+
const Completion& completion)
|
2788
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2789
|
+
// Start the underlying promise.
|
2790
|
+
void StartPromise(ClientMetadataHandle client_initial_metadata)
|
2791
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2792
|
+
// Publish status out to the application.
|
2793
|
+
void PublishStatus(
|
2794
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2795
|
+
ServerMetadataHandle trailing_metadata)
|
2796
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2797
|
+
// Publish server initial metadata out to the application.
|
2798
|
+
void PublishInitialMetadata(ServerMetadata* metadata)
|
2799
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2800
|
+
|
2801
|
+
ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
|
2802
|
+
Pipe<ServerMetadataHandle> server_initial_metadata_ ABSL_GUARDED_BY(mu()){
|
2803
|
+
arena()};
|
2804
|
+
Pipe<MessageHandle> client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2805
|
+
Pipe<MessageHandle> server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2806
|
+
|
2807
|
+
ClientMetadataHandle send_initial_metadata_;
|
2808
|
+
grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr;
|
2809
|
+
absl::variant<absl::monostate,
|
2810
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client,
|
2811
|
+
ServerMetadataHandle>
|
2812
|
+
recv_status_on_client_ ABSL_GUARDED_BY(mu());
|
2813
|
+
absl::optional<PipeReceiverNextType<ServerMetadataHandle>>
|
2814
|
+
server_initial_metadata_ready_;
|
2815
|
+
absl::optional<grpc_compression_algorithm> incoming_compression_algorithm_;
|
2816
|
+
Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu());
|
2817
|
+
Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu());
|
2818
|
+
Completion close_send_completion_ ABSL_GUARDED_BY(mu());
|
2819
|
+
bool is_trailers_only_ ABSL_GUARDED_BY(mu());
|
2820
|
+
};
|
2821
|
+
|
2822
|
+
void ClientPromiseBasedCall::StartPromise(
|
2823
|
+
ClientMetadataHandle client_initial_metadata) {
|
2824
|
+
GPR_ASSERT(!promise_.has_value());
|
2825
|
+
promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{
|
2826
|
+
std::move(client_initial_metadata),
|
2827
|
+
&server_initial_metadata_.sender,
|
2828
|
+
&client_to_server_messages_.receiver,
|
2829
|
+
&server_to_client_messages_.sender,
|
2830
|
+
});
|
2831
|
+
}
|
2832
|
+
|
2833
|
+
void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) {
|
2834
|
+
ScopedContext context(this);
|
2835
|
+
Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error)));
|
2836
|
+
}
|
2837
|
+
|
2838
|
+
grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
2839
|
+
size_t nops) const {
|
2840
|
+
BitSet<8> got_ops;
|
2841
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2842
|
+
const grpc_op& op = ops[op_idx];
|
2843
|
+
switch (op.op) {
|
2844
|
+
case GRPC_OP_SEND_INITIAL_METADATA:
|
2845
|
+
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
2846
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2847
|
+
}
|
2848
|
+
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
2849
|
+
op.data.send_initial_metadata.metadata)) {
|
2850
|
+
return GRPC_CALL_ERROR_INVALID_METADATA;
|
2851
|
+
}
|
2852
|
+
break;
|
2853
|
+
case GRPC_OP_SEND_MESSAGE:
|
2854
|
+
if (!AreWriteFlagsValid(op.flags)) {
|
2855
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2856
|
+
}
|
2857
|
+
break;
|
2858
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
2859
|
+
case GRPC_OP_RECV_MESSAGE:
|
2860
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2861
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
2862
|
+
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
2863
|
+
break;
|
2864
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2865
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2866
|
+
return GRPC_CALL_ERROR_NOT_ON_CLIENT;
|
2867
|
+
}
|
2868
|
+
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
2869
|
+
got_ops.set(op.op);
|
2870
|
+
}
|
2871
|
+
return GRPC_CALL_OK;
|
2872
|
+
}
|
2873
|
+
|
2874
|
+
void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
2875
|
+
const Completion& completion) {
|
2876
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2877
|
+
const grpc_op& op = ops[op_idx];
|
2878
|
+
switch (op.op) {
|
2879
|
+
case GRPC_OP_SEND_INITIAL_METADATA: {
|
2880
|
+
// compression not implemented
|
2881
|
+
GPR_ASSERT(
|
2882
|
+
!op.data.send_initial_metadata.maybe_compression_level.is_set);
|
2883
|
+
if (!completed()) {
|
2884
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
2885
|
+
op.data.send_initial_metadata.count,
|
2886
|
+
send_initial_metadata_.get());
|
2887
|
+
StartPromise(std::move(send_initial_metadata_));
|
2888
|
+
}
|
2889
|
+
} break;
|
2890
|
+
case GRPC_OP_RECV_INITIAL_METADATA: {
|
2891
|
+
recv_initial_metadata_ =
|
2892
|
+
op.data.recv_initial_metadata.recv_initial_metadata;
|
2893
|
+
server_initial_metadata_ready_.emplace(
|
2894
|
+
server_initial_metadata_.receiver.Next());
|
2895
|
+
recv_initial_metadata_completion_ =
|
2896
|
+
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata);
|
2897
|
+
} break;
|
2898
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
2899
|
+
recv_status_on_client_completion_ =
|
2900
|
+
AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient);
|
2901
|
+
if (auto* finished_metadata =
|
2902
|
+
absl::get_if<ServerMetadataHandle>(&recv_status_on_client_)) {
|
2903
|
+
PublishStatus(op.data.recv_status_on_client,
|
2904
|
+
std::move(*finished_metadata));
|
2905
|
+
} else {
|
2906
|
+
recv_status_on_client_ = op.data.recv_status_on_client;
|
2907
|
+
}
|
2908
|
+
} break;
|
2909
|
+
case GRPC_OP_SEND_MESSAGE:
|
2910
|
+
StartSendMessage(op, completion, &client_to_server_messages_.sender);
|
2911
|
+
break;
|
2912
|
+
case GRPC_OP_RECV_MESSAGE:
|
2913
|
+
StartRecvMessage(op, completion, &server_to_client_messages_.receiver);
|
2914
|
+
break;
|
2915
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2916
|
+
close_send_completion_ =
|
2917
|
+
AddOpToCompletion(completion, PendingOp::kSendCloseFromClient);
|
2918
|
+
GPR_ASSERT(close_send_completion_.has_value());
|
2919
|
+
break;
|
2920
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2921
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
2922
|
+
abort(); // unreachable
|
2923
|
+
}
|
2924
|
+
}
|
2925
|
+
}
|
2926
|
+
|
2927
|
+
grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
2928
|
+
size_t nops,
|
2929
|
+
void* notify_tag,
|
2930
|
+
bool is_notify_tag_closure) {
|
2931
|
+
MutexLock lock(mu());
|
2932
|
+
ScopedContext activity_context(this);
|
2933
|
+
if (nops == 0) {
|
2934
|
+
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
2935
|
+
return GRPC_CALL_OK;
|
2936
|
+
}
|
2937
|
+
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
2938
|
+
if (validation_result != GRPC_CALL_OK) {
|
2939
|
+
return validation_result;
|
2940
|
+
}
|
2941
|
+
Completion completion =
|
2942
|
+
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
2943
|
+
CommitBatch(ops, nops, completion);
|
2944
|
+
Update();
|
2945
|
+
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
2946
|
+
return GRPC_CALL_OK;
|
2947
|
+
}
|
2948
|
+
|
2949
|
+
void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) {
|
2950
|
+
incoming_compression_algorithm_ =
|
2951
|
+
metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
2952
|
+
server_initial_metadata_ready_.reset();
|
2953
|
+
GPR_ASSERT(recv_initial_metadata_ != nullptr);
|
2954
|
+
PublishMetadataArray(metadata,
|
2955
|
+
std::exchange(recv_initial_metadata_, nullptr));
|
2956
|
+
FinishOpOnCompletion(&recv_initial_metadata_completion_,
|
2957
|
+
PendingOp::kReceiveInitialMetadata);
|
2958
|
+
}
|
2959
|
+
|
2960
|
+
void ClientPromiseBasedCall::UpdateOnce() {
|
2961
|
+
if (grpc_call_trace.enabled()) {
|
2962
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s",
|
2963
|
+
DebugTag().c_str(),
|
2964
|
+
PresentAndCompletionText("server_initial_metadata_ready",
|
2965
|
+
server_initial_metadata_ready_.has_value(),
|
2966
|
+
recv_initial_metadata_completion_)
|
2967
|
+
.c_str(),
|
2968
|
+
PollStateDebugString().c_str(),
|
2969
|
+
promise_.has_value() ? "true" : "false");
|
2970
|
+
}
|
2971
|
+
if (server_initial_metadata_ready_.has_value()) {
|
2972
|
+
Poll<NextResult<ServerMetadataHandle>> r =
|
2973
|
+
(*server_initial_metadata_ready_)();
|
2974
|
+
if (auto* server_initial_metadata = r.value_if_ready()) {
|
2975
|
+
PublishInitialMetadata(server_initial_metadata->value().get());
|
2976
|
+
} else if (completed()) {
|
2977
|
+
ServerMetadata no_metadata{GetContext<Arena>()};
|
2978
|
+
PublishInitialMetadata(&no_metadata);
|
2979
|
+
}
|
2980
|
+
}
|
2981
|
+
if (!PollSendMessage()) {
|
2982
|
+
Finish(ServerMetadataFromStatus(absl::Status(
|
2983
|
+
absl::StatusCode::kInternal, "Failed to send message to server")));
|
2984
|
+
}
|
2985
|
+
if (!is_sending() && close_send_completion_.has_value()) {
|
2986
|
+
client_to_server_messages_.sender.Close();
|
2987
|
+
FinishOpOnCompletion(&close_send_completion_,
|
2988
|
+
PendingOp::kSendCloseFromClient);
|
2989
|
+
}
|
2990
|
+
if (promise_.has_value()) {
|
2991
|
+
Poll<ServerMetadataHandle> r = promise_();
|
2992
|
+
if (grpc_call_trace.enabled()) {
|
2993
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
|
2994
|
+
DebugTag().c_str(),
|
2995
|
+
PollToString(r, [](const ServerMetadataHandle& h) {
|
2996
|
+
return h->DebugString();
|
2997
|
+
}).c_str());
|
2998
|
+
}
|
2999
|
+
if (auto* result = r.value_if_ready()) {
|
3000
|
+
AcceptTransportStatsFromContext();
|
3001
|
+
Finish(std::move(*result));
|
3002
|
+
}
|
3003
|
+
}
|
3004
|
+
if (incoming_compression_algorithm_.has_value()) {
|
3005
|
+
PollRecvMessage(*incoming_compression_algorithm_);
|
3006
|
+
}
|
3007
|
+
}
|
3008
|
+
|
3009
|
+
void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
3010
|
+
if (grpc_call_trace.enabled()) {
|
3011
|
+
gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
|
3012
|
+
trailing_metadata->DebugString().c_str());
|
3013
|
+
}
|
3014
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3015
|
+
ResetDeadline();
|
3016
|
+
set_completed();
|
3017
|
+
if (recv_initial_metadata_ != nullptr) {
|
3018
|
+
ForceImmediateRepoll();
|
3019
|
+
}
|
3020
|
+
const bool pending_initial_metadata =
|
3021
|
+
server_initial_metadata_ready_.has_value();
|
3022
|
+
if (!pending_initial_metadata) {
|
3023
|
+
server_initial_metadata_ready_.emplace(
|
3024
|
+
server_initial_metadata_.receiver.Next());
|
3025
|
+
}
|
3026
|
+
Poll<NextResult<ServerMetadataHandle>> r =
|
3027
|
+
(*server_initial_metadata_ready_)();
|
3028
|
+
server_initial_metadata_ready_.reset();
|
3029
|
+
if (auto* result = r.value_if_ready()) {
|
3030
|
+
if (pending_initial_metadata) PublishInitialMetadata(result->value().get());
|
3031
|
+
is_trailers_only_ = false;
|
3032
|
+
} else {
|
3033
|
+
if (pending_initial_metadata) {
|
3034
|
+
ServerMetadata no_metadata{GetContext<Arena>()};
|
3035
|
+
PublishInitialMetadata(&no_metadata);
|
3036
|
+
}
|
3037
|
+
is_trailers_only_ = true;
|
3038
|
+
}
|
3039
|
+
if (auto* channelz_channel = channel()->channelz_node()) {
|
3040
|
+
if (trailing_metadata->get(GrpcStatusMetadata())
|
3041
|
+
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3042
|
+
channelz_channel->RecordCallSucceeded();
|
3043
|
+
} else {
|
3044
|
+
channelz_channel->RecordCallFailed();
|
3045
|
+
}
|
3046
|
+
}
|
3047
|
+
if (auto* status_request =
|
3048
|
+
absl::get_if<grpc_op::grpc_op_data::grpc_op_recv_status_on_client>(
|
3049
|
+
&recv_status_on_client_)) {
|
3050
|
+
PublishStatus(*status_request, std::move(trailing_metadata));
|
3051
|
+
} else {
|
3052
|
+
recv_status_on_client_ = std::move(trailing_metadata);
|
3053
|
+
}
|
3054
|
+
}
|
3055
|
+
|
3056
|
+
namespace {
|
3057
|
+
std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
|
3058
|
+
std::string out = absl::StrCat(
|
3059
|
+
trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
|
3060
|
+
? "Error received from peer"
|
3061
|
+
: "Error generated by client",
|
3062
|
+
"grpc_status: ",
|
3063
|
+
grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
|
3064
|
+
.value_or(GRPC_STATUS_UNKNOWN)));
|
3065
|
+
if (const Slice* message =
|
3066
|
+
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3067
|
+
absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
|
3068
|
+
}
|
3069
|
+
if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
|
3070
|
+
absl::StrAppend(&out, "\nStatus Context:");
|
3071
|
+
for (const std::string& annotation : *annotations) {
|
3072
|
+
absl::StrAppend(&out, "\n ", annotation);
|
3073
|
+
}
|
3074
|
+
}
|
3075
|
+
return out;
|
3076
|
+
}
|
3077
|
+
} // namespace
|
3078
|
+
|
3079
|
+
void ClientPromiseBasedCall::PublishStatus(
|
3080
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
3081
|
+
ServerMetadataHandle trailing_metadata) {
|
3082
|
+
const grpc_status_code status = trailing_metadata->get(GrpcStatusMetadata())
|
3083
|
+
.value_or(GRPC_STATUS_UNKNOWN);
|
3084
|
+
*op_args.status = status;
|
3085
|
+
absl::string_view message_string;
|
3086
|
+
if (Slice* message = trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
3087
|
+
message_string = message->as_string_view();
|
3088
|
+
*op_args.status_details = message->Ref().TakeCSlice();
|
3089
|
+
} else {
|
3090
|
+
*op_args.status_details = grpc_empty_slice();
|
3091
|
+
}
|
3092
|
+
if (message_string.empty()) {
|
3093
|
+
RunFinalization(status, nullptr);
|
3094
|
+
} else {
|
3095
|
+
std::string error_string(message_string);
|
3096
|
+
RunFinalization(status, error_string.c_str());
|
3097
|
+
}
|
3098
|
+
if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
|
3099
|
+
*op_args.error_string =
|
3100
|
+
gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
|
3101
|
+
}
|
3102
|
+
PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata);
|
3103
|
+
// Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding
|
3104
|
+
// (so we don't call through twice)
|
3105
|
+
recv_status_on_client_ = absl::monostate();
|
3106
|
+
FinishOpOnCompletion(&recv_status_on_client_completion_,
|
3107
|
+
PendingOp::kReceiveStatusOnClient);
|
3108
|
+
}
|
3109
|
+
#endif
|
3110
|
+
|
3111
|
+
///////////////////////////////////////////////////////////////////////////////
|
3112
|
+
// ServerPromiseBasedCall
|
3113
|
+
|
3114
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3115
|
+
|
3116
|
+
class ServerPromiseBasedCall final : public PromiseBasedCall {
|
3117
|
+
public:
|
3118
|
+
ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
|
3119
|
+
|
3120
|
+
void CancelWithErrorLocked(grpc_error_handle) override
|
3121
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3122
|
+
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3123
|
+
bool is_notify_tag_closure) override;
|
3124
|
+
bool failed_before_recv_message() const override { abort(); }
|
3125
|
+
bool is_trailers_only() const override { abort(); }
|
3126
|
+
absl::string_view GetServerAuthority() const override { return ""; }
|
3127
|
+
|
3128
|
+
// Polling order for the server promise stack:
|
3129
|
+
//
|
3130
|
+
// │ ┌───────────────────────────────────────┐
|
3131
|
+
// │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management,
|
3132
|
+
// │ ├───────────────────────────────────────┤ signal call end to app
|
3133
|
+
// │ │ ConnectedChannel ├─┐
|
3134
|
+
// │ ├───────────────────────────────────────┤ └► Interactions with the
|
3135
|
+
// │ │ ... closest to transport filter │ transport - send/recv msgs
|
3136
|
+
// │ ├───────────────────────────────────────┤ and metadata, call phase
|
3137
|
+
// │ │ ... │ ordering
|
3138
|
+
// │ ├───────────────────────────────────────┤
|
3139
|
+
// │ │ ... closest to app filter │ ┌► Request matching, initial
|
3140
|
+
// │ ├───────────────────────────────────────┤ │ setup, publishing call to
|
3141
|
+
// │ │ Server::ChannelData::MakeCallPromise ├─┘ application
|
3142
|
+
// │ ├───────────────────────────────────────┤
|
3143
|
+
// │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions,
|
3144
|
+
// ▼ └───────────────────────────────────────┘ forwarding messages,
|
3145
|
+
// Polling & sending trailing metadata
|
3146
|
+
// instantiation
|
3147
|
+
// order
|
3148
|
+
|
3149
|
+
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
3150
|
+
Poll<ServerMetadataHandle> PollTopOfCall()
|
3151
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3152
|
+
|
3153
|
+
std::string DebugTag() const override {
|
3154
|
+
return absl::StrFormat("SERVER_CALL[%p]: ", this);
|
3155
|
+
}
|
3156
|
+
|
3157
|
+
ServerCallContext* server_call_context() override { return &call_context_; }
|
3158
|
+
|
3159
|
+
private:
|
3160
|
+
class RecvCloseOpCancelState {
|
3161
|
+
public:
|
3162
|
+
// Request that receiver be filled in per grpc_op_recv_close_on_server.
|
3163
|
+
// Returns true if the request can be fulfilled immediately.
|
3164
|
+
// Returns false if the request will be fulfilled later.
|
3165
|
+
bool ReceiveCloseOnServerOpStarted(int* receiver) {
|
3166
|
+
switch (state_) {
|
3167
|
+
case kUnset:
|
3168
|
+
state_ = reinterpret_cast<uintptr_t>(receiver);
|
3169
|
+
return false;
|
3170
|
+
case kFinishedWithFailure:
|
3171
|
+
*receiver = 1;
|
3172
|
+
return true;
|
3173
|
+
case kFinishedWithSuccess:
|
3174
|
+
*receiver = 0;
|
3175
|
+
return true;
|
3176
|
+
default:
|
3177
|
+
abort(); // unreachable
|
3178
|
+
}
|
3179
|
+
}
|
3180
|
+
|
3181
|
+
// Mark the call as having completed.
|
3182
|
+
// Returns true if this finishes a previous RequestReceiveCloseOnServer.
|
3183
|
+
bool CompleteCall(bool success) {
|
3184
|
+
switch (state_) {
|
3185
|
+
case kUnset:
|
3186
|
+
state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
|
3187
|
+
return false;
|
3188
|
+
case kFinishedWithFailure:
|
3189
|
+
case kFinishedWithSuccess:
|
3190
|
+
abort(); // unreachable
|
3191
|
+
default:
|
3192
|
+
*reinterpret_cast<int*>(state_) = success ? 0 : 1;
|
3193
|
+
state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
|
3194
|
+
return true;
|
3195
|
+
}
|
3196
|
+
}
|
3197
|
+
|
3198
|
+
std::string ToString() const {
|
3199
|
+
switch (state_) {
|
3200
|
+
case kUnset:
|
3201
|
+
return "Unset";
|
3202
|
+
case kFinishedWithFailure:
|
3203
|
+
return "FinishedWithFailure";
|
3204
|
+
case kFinishedWithSuccess:
|
3205
|
+
return "FinishedWithSuccess";
|
3206
|
+
default:
|
3207
|
+
return absl::StrFormat("WaitingForReceiver(%p)",
|
3208
|
+
reinterpret_cast<void*>(state_));
|
3209
|
+
}
|
3210
|
+
}
|
3211
|
+
|
3212
|
+
private:
|
3213
|
+
static constexpr uintptr_t kUnset = 0;
|
3214
|
+
static constexpr uintptr_t kFinishedWithFailure = 1;
|
3215
|
+
static constexpr uintptr_t kFinishedWithSuccess = 2;
|
3216
|
+
// Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess
|
3217
|
+
// OR an int* that wants to receive the final status.
|
3218
|
+
uintptr_t state_ = kUnset;
|
3219
|
+
};
|
3220
|
+
|
3221
|
+
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
3222
|
+
void CommitBatch(const grpc_op* ops, size_t nops,
|
3223
|
+
const Completion& completion)
|
3224
|
+
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3225
|
+
|
3226
|
+
friend class ServerCallContext;
|
3227
|
+
ServerCallContext call_context_;
|
3228
|
+
Server* const server_;
|
3229
|
+
ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
|
3230
|
+
PipeSender<MessageHandle>* server_to_client_messages_ ABSL_GUARDED_BY(mu()) =
|
3231
|
+
nullptr;
|
3232
|
+
PipeReceiver<MessageHandle>* client_to_server_messages_
|
3233
|
+
ABSL_GUARDED_BY(mu()) = nullptr;
|
3234
|
+
using SendInitialMetadataState =
|
3235
|
+
absl::variant<absl::monostate, PipeSender<ServerMetadataHandle>*,
|
3236
|
+
typename PipeSender<ServerMetadataHandle>::PushType>;
|
3237
|
+
SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) =
|
3238
|
+
absl::monostate{};
|
3239
|
+
ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu());
|
3240
|
+
grpc_compression_algorithm incoming_compression_algorithm_
|
3241
|
+
ABSL_GUARDED_BY(mu());
|
3242
|
+
RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu());
|
3243
|
+
Completion recv_close_completion_ ABSL_GUARDED_BY(mu());
|
3244
|
+
bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false;
|
3245
|
+
Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu());
|
3246
|
+
ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu());
|
3247
|
+
};
|
3248
|
+
|
3249
|
+
ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
3250
|
+
grpc_call_create_args* args)
|
3251
|
+
: PromiseBasedCall(arena, 0, *args),
|
3252
|
+
call_context_(this, args->server_transport_data),
|
3253
|
+
server_(args->server) {
|
3254
|
+
global_stats().IncrementServerCallsCreated();
|
3255
|
+
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3256
|
+
if (channelz_node != nullptr) {
|
3257
|
+
channelz_node->RecordCallStarted();
|
3258
|
+
}
|
3259
|
+
MutexLock lock(mu());
|
3260
|
+
ScopedContext activity_context(this);
|
3261
|
+
promise_ = channel()->channel_stack()->MakeServerCallPromise(
|
3262
|
+
CallArgs{nullptr, nullptr, nullptr, nullptr});
|
3263
|
+
}
|
3264
|
+
|
3265
|
+
Poll<ServerMetadataHandle> ServerPromiseBasedCall::PollTopOfCall() {
|
3266
|
+
if (grpc_call_trace.enabled()) {
|
3267
|
+
gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(),
|
3268
|
+
cancel_send_and_receive_ ? "force-" : "",
|
3269
|
+
send_trailing_metadata_ != nullptr
|
3270
|
+
? absl::StrCat("send-metadata:",
|
3271
|
+
send_trailing_metadata_->DebugString(), " ")
|
3272
|
+
.c_str()
|
3273
|
+
: " ",
|
3274
|
+
PollStateDebugString().c_str());
|
3275
|
+
}
|
3276
|
+
|
3277
|
+
if (cancel_send_and_receive_) {
|
3278
|
+
CancelSendMessage();
|
3279
|
+
CancelRecvMessage();
|
3280
|
+
}
|
3281
|
+
|
3282
|
+
PollSendMessage();
|
3283
|
+
PollRecvMessage(incoming_compression_algorithm_);
|
3284
|
+
|
3285
|
+
if (!is_sending() && send_trailing_metadata_ != nullptr) {
|
3286
|
+
server_to_client_messages_->Close();
|
3287
|
+
return std::move(send_trailing_metadata_);
|
3288
|
+
}
|
3289
|
+
|
3290
|
+
return Pending{};
|
3291
|
+
}
|
3292
|
+
|
3293
|
+
void ServerPromiseBasedCall::UpdateOnce() {
|
3294
|
+
if (grpc_call_trace.enabled()) {
|
3295
|
+
gpr_log(
|
3296
|
+
GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s",
|
3297
|
+
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3298
|
+
recv_close_completion_.has_value()
|
3299
|
+
? absl::StrCat(":", CompletionString(recv_close_completion_))
|
3300
|
+
.c_str()
|
3301
|
+
: "",
|
3302
|
+
send_status_from_server_completion_.has_value()
|
3303
|
+
? absl::StrCat(
|
3304
|
+
"send_status:",
|
3305
|
+
CompletionString(send_status_from_server_completion_), " ")
|
3306
|
+
.c_str()
|
3307
|
+
: "",
|
3308
|
+
PollStateDebugString().c_str(),
|
3309
|
+
promise_.has_value() ? "true" : "false");
|
3310
|
+
}
|
3311
|
+
if (auto* p =
|
3312
|
+
absl::get_if<typename PipeSender<ServerMetadataHandle>::PushType>(
|
3313
|
+
&send_initial_metadata_state_)) {
|
3314
|
+
if ((*p)().ready()) {
|
3315
|
+
send_initial_metadata_state_ = absl::monostate{};
|
3316
|
+
}
|
3317
|
+
}
|
3318
|
+
if (promise_.has_value()) {
|
3319
|
+
auto r = promise_();
|
3320
|
+
if (grpc_call_trace.enabled()) {
|
3321
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
|
3322
|
+
DebugTag().c_str(),
|
3323
|
+
PollToString(r, [](const ServerMetadataHandle& h) {
|
3324
|
+
return h->DebugString();
|
3325
|
+
}).c_str());
|
3326
|
+
}
|
3327
|
+
if (auto* result = r.value_if_ready()) {
|
3328
|
+
if (grpc_call_trace.enabled()) {
|
3329
|
+
gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s",
|
3330
|
+
DebugTag().c_str(),
|
3331
|
+
recv_close_op_cancel_state_.ToString().c_str(),
|
3332
|
+
(*result)->DebugString().c_str());
|
3333
|
+
}
|
3334
|
+
if (recv_close_op_cancel_state_.CompleteCall(
|
3335
|
+
(*result)->get(GrpcStatusFromWire()).value_or(false))) {
|
3336
|
+
FinishOpOnCompletion(&recv_close_completion_,
|
3337
|
+
PendingOp::kReceiveCloseOnServer);
|
3338
|
+
}
|
3339
|
+
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3340
|
+
if (channelz_node != nullptr) {
|
3341
|
+
if ((*result)
|
3342
|
+
->get(GrpcStatusMetadata())
|
3343
|
+
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3344
|
+
channelz_node->RecordCallSucceeded();
|
3345
|
+
} else {
|
3346
|
+
channelz_node->RecordCallFailed();
|
3347
|
+
}
|
3348
|
+
}
|
3349
|
+
if (send_status_from_server_completion_.has_value()) {
|
3350
|
+
FinishOpOnCompletion(&send_status_from_server_completion_,
|
3351
|
+
PendingOp::kSendStatusFromServer);
|
3352
|
+
}
|
3353
|
+
CancelSendMessage();
|
3354
|
+
CancelRecvMessage();
|
3355
|
+
set_completed();
|
3356
|
+
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3357
|
+
}
|
3358
|
+
}
|
3359
|
+
}
|
3360
|
+
|
3361
|
+
grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
3362
|
+
size_t nops) const {
|
3363
|
+
BitSet<8> got_ops;
|
3364
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3365
|
+
const grpc_op& op = ops[op_idx];
|
3366
|
+
switch (op.op) {
|
3367
|
+
case GRPC_OP_SEND_INITIAL_METADATA:
|
3368
|
+
if (!AreInitialMetadataFlagsValid(op.flags)) {
|
3369
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3370
|
+
}
|
3371
|
+
if (!ValidateMetadata(op.data.send_initial_metadata.count,
|
3372
|
+
op.data.send_initial_metadata.metadata)) {
|
3373
|
+
return GRPC_CALL_ERROR_INVALID_METADATA;
|
3374
|
+
}
|
3375
|
+
break;
|
3376
|
+
case GRPC_OP_SEND_MESSAGE:
|
3377
|
+
if (!AreWriteFlagsValid(op.flags)) {
|
3378
|
+
return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3379
|
+
}
|
3380
|
+
break;
|
3381
|
+
case GRPC_OP_RECV_MESSAGE:
|
3382
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3383
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3384
|
+
if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
|
3385
|
+
break;
|
3386
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
3387
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3388
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3389
|
+
return GRPC_CALL_ERROR_NOT_ON_SERVER;
|
3390
|
+
}
|
3391
|
+
if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
3392
|
+
got_ops.set(op.op);
|
3393
|
+
}
|
3394
|
+
return GRPC_CALL_OK;
|
3395
|
+
}
|
3396
|
+
|
3397
|
+
void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
3398
|
+
const Completion& completion) {
|
3399
|
+
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3400
|
+
const grpc_op& op = ops[op_idx];
|
3401
|
+
switch (op.op) {
|
3402
|
+
case GRPC_OP_SEND_INITIAL_METADATA: {
|
3403
|
+
// compression not implemented
|
3404
|
+
GPR_ASSERT(
|
3405
|
+
!op.data.send_initial_metadata.maybe_compression_level.is_set);
|
3406
|
+
if (!completed()) {
|
3407
|
+
auto metadata = arena()->MakePooled<ServerMetadata>(arena());
|
3408
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
3409
|
+
op.data.send_initial_metadata.count, metadata.get());
|
3410
|
+
if (grpc_call_trace.enabled()) {
|
3411
|
+
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3412
|
+
DebugTag().c_str());
|
3413
|
+
}
|
3414
|
+
auto* pipe = absl::get<PipeSender<ServerMetadataHandle>*>(
|
3415
|
+
send_initial_metadata_state_);
|
3416
|
+
send_initial_metadata_state_ = pipe->Push(std::move(metadata));
|
3417
|
+
}
|
3418
|
+
} break;
|
3419
|
+
case GRPC_OP_SEND_MESSAGE:
|
3420
|
+
StartSendMessage(op, completion, server_to_client_messages_);
|
3421
|
+
break;
|
3422
|
+
case GRPC_OP_RECV_MESSAGE:
|
3423
|
+
StartRecvMessage(op, completion, client_to_server_messages_);
|
3424
|
+
break;
|
3425
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3426
|
+
send_trailing_metadata_ = arena()->MakePooled<ServerMetadata>(arena());
|
3427
|
+
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3428
|
+
op.data.send_status_from_server.trailing_metadata_count,
|
3429
|
+
send_trailing_metadata_.get());
|
3430
|
+
send_trailing_metadata_->Set(GrpcStatusMetadata(),
|
3431
|
+
op.data.send_status_from_server.status);
|
3432
|
+
if (auto* details = op.data.send_status_from_server.status_details) {
|
3433
|
+
send_trailing_metadata_->Set(GrpcMessageMetadata(),
|
3434
|
+
Slice(CSliceRef(*details)));
|
3435
|
+
}
|
3436
|
+
send_status_from_server_completion_ =
|
3437
|
+
AddOpToCompletion(completion, PendingOp::kSendStatusFromServer);
|
3438
|
+
break;
|
3439
|
+
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3440
|
+
if (grpc_call_trace.enabled()) {
|
3441
|
+
gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
|
3442
|
+
DebugTag().c_str(),
|
3443
|
+
recv_close_op_cancel_state_.ToString().c_str());
|
3444
|
+
}
|
3445
|
+
if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
|
3446
|
+
op.data.recv_close_on_server.cancelled)) {
|
3447
|
+
recv_close_completion_ =
|
3448
|
+
AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
|
3449
|
+
}
|
3450
|
+
break;
|
3451
|
+
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
3452
|
+
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
3453
|
+
case GRPC_OP_RECV_INITIAL_METADATA:
|
3454
|
+
abort(); // unreachable
|
3455
|
+
}
|
3456
|
+
}
|
3457
|
+
}
|
3458
|
+
|
3459
|
+
grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
3460
|
+
size_t nops,
|
3461
|
+
void* notify_tag,
|
3462
|
+
bool is_notify_tag_closure) {
|
3463
|
+
MutexLock lock(mu());
|
3464
|
+
ScopedContext activity_context(this);
|
3465
|
+
if (nops == 0) {
|
3466
|
+
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3467
|
+
return GRPC_CALL_OK;
|
3468
|
+
}
|
3469
|
+
const grpc_call_error validation_result = ValidateBatch(ops, nops);
|
3470
|
+
if (validation_result != GRPC_CALL_OK) {
|
3471
|
+
return validation_result;
|
3472
|
+
}
|
3473
|
+
Completion completion =
|
3474
|
+
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
3475
|
+
CommitBatch(ops, nops, completion);
|
3476
|
+
Update();
|
3477
|
+
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
3478
|
+
return GRPC_CALL_OK;
|
3479
|
+
}
|
3480
|
+
|
3481
|
+
void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) {
|
3482
|
+
if (!promise_.has_value()) return;
|
3483
|
+
cancel_send_and_receive_ = true;
|
3484
|
+
send_trailing_metadata_ = ServerMetadataFromStatus(error, arena());
|
3485
|
+
ForceWakeup();
|
3486
|
+
}
|
3487
|
+
|
3488
|
+
#endif
|
3489
|
+
|
3490
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3491
|
+
ArenaPromise<ServerMetadataHandle>
|
3492
|
+
ServerCallContext::MakeTopOfServerCallPromise(
|
3493
|
+
CallArgs call_args, grpc_completion_queue* cq,
|
3494
|
+
grpc_metadata_array* publish_initial_metadata,
|
3495
|
+
absl::FunctionRef<void(grpc_call* call)> publish) {
|
3496
|
+
call_->mu()->AssertHeld();
|
3497
|
+
call_->SetCompletionQueueLocked(cq);
|
3498
|
+
call_->server_to_client_messages_ = call_args.server_to_client_messages;
|
3499
|
+
call_->client_to_server_messages_ = call_args.client_to_server_messages;
|
3500
|
+
call_->send_initial_metadata_state_ = call_args.server_initial_metadata;
|
3501
|
+
call_->incoming_compression_algorithm_ =
|
3502
|
+
call_args.client_initial_metadata->get(GrpcEncodingMetadata())
|
3503
|
+
.value_or(GRPC_COMPRESS_NONE);
|
3504
|
+
call_->client_initial_metadata_ =
|
3505
|
+
std::move(call_args.client_initial_metadata);
|
3506
|
+
PublishMetadataArray(call_->client_initial_metadata_.get(),
|
3507
|
+
publish_initial_metadata);
|
3508
|
+
call_->ExternalRef();
|
3509
|
+
publish(call_->c_ptr());
|
3510
|
+
return [this]() {
|
3511
|
+
call_->mu()->AssertHeld();
|
3512
|
+
return call_->PollTopOfCall();
|
3513
|
+
};
|
3514
|
+
}
|
3515
|
+
#else
|
3516
|
+
ArenaPromise<ServerMetadataHandle>
|
3517
|
+
ServerCallContext::MakeTopOfServerCallPromise(
|
3518
|
+
CallArgs, grpc_completion_queue*, grpc_metadata_array*,
|
3519
|
+
absl::FunctionRef<void(grpc_call*)>) {
|
3520
|
+
(void)call_;
|
3521
|
+
Crash("Promise-based server call is not enabled");
|
3522
|
+
}
|
3523
|
+
#endif
|
3524
|
+
|
3525
|
+
} // namespace grpc_core
|
3526
|
+
|
3527
|
+
///////////////////////////////////////////////////////////////////////////////
|
3528
|
+
// C-based API
|
3529
|
+
|
3530
|
+
void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
|
3531
|
+
grpc_core::ExecCtx exec_ctx;
|
3532
|
+
return grpc_core::Call::FromC(call)->arena()->Alloc(size);
|
3533
|
+
}
|
3534
|
+
|
3535
|
+
size_t grpc_call_get_initial_size_estimate() {
|
3536
|
+
return grpc_core::FilterStackCall::InitialSizeEstimate();
|
3537
|
+
}
|
3538
|
+
|
3539
|
+
grpc_error_handle grpc_call_create(grpc_call_create_args* args,
|
3540
|
+
grpc_call** out_call) {
|
3541
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
|
3542
|
+
if (grpc_core::IsPromiseBasedClientCallEnabled() &&
|
3543
|
+
args->server_transport_data == nullptr && args->channel->is_promising()) {
|
3544
|
+
return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
|
3545
|
+
args, out_call);
|
3546
|
+
}
|
3547
|
+
#endif
|
3548
|
+
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
3549
|
+
if (grpc_core::IsPromiseBasedServerCallEnabled() &&
|
3550
|
+
args->server_transport_data != nullptr && args->channel->is_promising()) {
|
3551
|
+
return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
|
3552
|
+
args, out_call);
|
3553
|
+
}
|
3554
|
+
#endif
|
3555
|
+
return grpc_core::FilterStackCall::Create(args, out_call);
|
3556
|
+
}
|
3557
|
+
|
3558
|
+
void grpc_call_set_completion_queue(grpc_call* call,
|
3559
|
+
grpc_completion_queue* cq) {
|
3560
|
+
grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
|
3561
|
+
}
|
3562
|
+
|
3563
|
+
void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
|
3564
|
+
|
3565
|
+
void grpc_call_unref(grpc_call* c) {
|
3566
|
+
grpc_core::ExecCtx exec_ctx;
|
3567
|
+
grpc_core::Call::FromC(c)->ExternalUnref();
|
3568
|
+
}
|
3569
|
+
|
3570
|
+
char* grpc_call_get_peer(grpc_call* call) {
|
3571
|
+
return grpc_core::Call::FromC(call)->GetPeer();
|
3572
|
+
}
|
3573
|
+
|
3574
|
+
grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element) {
|
3575
|
+
return grpc_core::FilterStackCall::FromTopElem(surface_element)->c_ptr();
|
3576
|
+
}
|
3577
|
+
|
3578
|
+
grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
|
3579
|
+
GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved));
|
3580
|
+
GPR_ASSERT(reserved == nullptr);
|
3581
|
+
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
3582
|
+
grpc_core::ExecCtx exec_ctx;
|
3583
|
+
grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
|
3584
|
+
return GRPC_CALL_OK;
|
3585
|
+
}
|
3586
|
+
|
3587
|
+
grpc_call_error grpc_call_cancel_with_status(grpc_call* c,
|
3588
|
+
grpc_status_code status,
|
3589
|
+
const char* description,
|
3590
|
+
void* reserved) {
|
3591
|
+
GRPC_API_TRACE(
|
3592
|
+
"grpc_call_cancel_with_status("
|
3593
|
+
"c=%p, status=%d, description=%s, reserved=%p)",
|
3594
|
+
4, (c, (int)status, description, reserved));
|
3595
|
+
GPR_ASSERT(reserved == nullptr);
|
3596
|
+
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
3597
|
+
grpc_core::ExecCtx exec_ctx;
|
3598
|
+
grpc_core::Call::FromC(c)->CancelWithStatus(status, description);
|
3599
|
+
return GRPC_CALL_OK;
|
3600
|
+
}
|
3601
|
+
|
3602
|
+
void grpc_call_cancel_internal(grpc_call* call) {
|
3603
|
+
grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
|
3604
|
+
}
|
3605
|
+
|
3606
|
+
grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
|
3607
|
+
grpc_call* call) {
|
3608
|
+
return grpc_core::Call::FromC(call)->test_only_compression_algorithm();
|
3609
|
+
}
|
3610
|
+
|
3611
|
+
uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
|
3612
|
+
return grpc_core::Call::FromC(call)->test_only_message_flags();
|
3613
|
+
}
|
3614
|
+
|
3615
|
+
uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) {
|
3616
|
+
return grpc_core::Call::FromC(call)->test_only_encodings_accepted_by_peer();
|
3617
|
+
}
|
3618
|
+
|
3619
|
+
grpc_core::Arena* grpc_call_get_arena(grpc_call* call) {
|
3620
|
+
return grpc_core::Call::FromC(call)->arena();
|
3621
|
+
}
|
3622
|
+
|
3623
|
+
grpc_call_stack* grpc_call_get_call_stack(grpc_call* call) {
|
3624
|
+
return grpc_core::Call::FromC(call)->call_stack();
|
3625
|
+
}
|
3626
|
+
|
3627
|
+
grpc_call_error grpc_call_start_batch(grpc_call* call, const grpc_op* ops,
|
3628
|
+
size_t nops, void* tag, void* reserved) {
|
1758
3629
|
GRPC_API_TRACE(
|
1759
3630
|
"grpc_call_start_batch(call=%p, ops=%p, nops=%lu, tag=%p, "
|
1760
3631
|
"reserved=%p)",
|
1761
3632
|
5, (call, ops, (unsigned long)nops, tag, reserved));
|
1762
3633
|
|
1763
3634
|
if (reserved != nullptr) {
|
1764
|
-
|
3635
|
+
return GRPC_CALL_ERROR;
|
1765
3636
|
} else {
|
1766
3637
|
grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
|
1767
3638
|
grpc_core::ExecCtx exec_ctx;
|
1768
|
-
|
3639
|
+
return grpc_core::Call::FromC(call)->StartBatch(ops, nops, tag, false);
|
1769
3640
|
}
|
1770
|
-
|
1771
|
-
return err;
|
1772
3641
|
}
|
1773
3642
|
|
1774
3643
|
grpc_call_error grpc_call_start_batch_and_execute(grpc_call* call,
|
1775
3644
|
const grpc_op* ops,
|
1776
3645
|
size_t nops,
|
1777
3646
|
grpc_closure* closure) {
|
1778
|
-
return
|
3647
|
+
return grpc_core::Call::FromC(call)->StartBatch(ops, nops, closure, true);
|
1779
3648
|
}
|
1780
3649
|
|
1781
3650
|
void grpc_call_context_set(grpc_call* call, grpc_context_index elem,
|
1782
3651
|
void* value, void (*destroy)(void* value)) {
|
1783
|
-
|
1784
|
-
call->context[elem].destroy(call->context[elem].value);
|
1785
|
-
}
|
1786
|
-
call->context[elem].value = value;
|
1787
|
-
call->context[elem].destroy = destroy;
|
3652
|
+
return grpc_core::Call::FromC(call)->ContextSet(elem, value, destroy);
|
1788
3653
|
}
|
1789
3654
|
|
1790
3655
|
void* grpc_call_context_get(grpc_call* call, grpc_context_index elem) {
|
1791
|
-
return call->
|
3656
|
+
return grpc_core::Call::FromC(call)->ContextGet(elem);
|
1792
3657
|
}
|
1793
3658
|
|
1794
|
-
uint8_t grpc_call_is_client(grpc_call* call) {
|
3659
|
+
uint8_t grpc_call_is_client(grpc_call* call) {
|
3660
|
+
return grpc_core::Call::FromC(call)->is_client();
|
3661
|
+
}
|
1795
3662
|
|
1796
3663
|
grpc_compression_algorithm grpc_call_compression_for_level(
|
1797
3664
|
grpc_call* call, grpc_compression_level level) {
|
1798
|
-
return call->
|
3665
|
+
return grpc_core::Call::FromC(call)->compression_for_level(level);
|
1799
3666
|
}
|
1800
3667
|
|
1801
3668
|
bool grpc_call_is_trailers_only(const grpc_call* call) {
|
1802
|
-
|
1803
|
-
GPR_DEBUG_ASSERT(!result || call->recv_initial_metadata.empty());
|
1804
|
-
return result;
|
3669
|
+
return grpc_core::Call::FromC(call)->is_trailers_only();
|
1805
3670
|
}
|
1806
3671
|
|
1807
3672
|
int grpc_call_failed_before_recv_message(const grpc_call* c) {
|
1808
|
-
return c->
|
3673
|
+
return grpc_core::Call::FromC(c)->failed_before_recv_message();
|
3674
|
+
}
|
3675
|
+
|
3676
|
+
absl::string_view grpc_call_server_authority(const grpc_call* call) {
|
3677
|
+
return grpc_core::Call::FromC(call)->GetServerAuthority();
|
1809
3678
|
}
|
1810
3679
|
|
1811
3680
|
const char* grpc_call_error_to_string(grpc_call_error error) {
|