grpc 1.53.0.pre2 → 1.54.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of grpc might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Makefile +80 -66
- data/include/grpc/event_engine/event_engine.h +30 -14
- data/include/grpc/grpc_security.h +4 -0
- data/include/grpc/impl/grpc_types.h +11 -2
- data/include/grpc/support/port_platform.h +4 -4
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +11 -0
- data/src/core/ext/filters/client_channel/backend_metric.cc +6 -0
- data/src/core/ext/filters/client_channel/backup_poller.cc +2 -11
- data/src/core/ext/filters/client_channel/backup_poller.h +0 -3
- data/src/core/ext/filters/client_channel/client_channel.cc +848 -813
- data/src/core/ext/filters/client_channel/client_channel.h +131 -173
- data/src/core/ext/filters/client_channel/client_channel_internal.h +114 -0
- data/src/core/ext/filters/client_channel/config_selector.h +4 -3
- data/src/core/ext/filters/client_channel/http_proxy.cc +1 -1
- data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +6 -1
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +17 -18
- data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +134 -151
- data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +1 -15
- data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +14 -10
- data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc +68 -30
- data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +13 -5
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +8 -1
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +2 -5
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +2 -2
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +30 -38
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +4 -4
- data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +20 -26
- data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +31 -179
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +1 -2
- data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +1 -2
- data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +4 -2
- data/src/core/ext/filters/client_channel/retry_filter.cc +95 -102
- data/src/core/ext/filters/client_channel/subchannel.cc +2 -4
- data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +26 -27
- data/src/core/ext/filters/client_channel/subchannel_stream_client.h +8 -5
- data/src/core/ext/filters/http/client/http_client_filter.cc +3 -3
- data/src/core/ext/filters/http/http_filters_plugin.cc +1 -12
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +27 -11
- data/src/core/ext/filters/message_size/message_size_filter.cc +141 -224
- data/src/core/ext/filters/message_size/message_size_filter.h +48 -3
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +7 -6
- data/src/core/ext/gcp/metadata_query.cc +142 -0
- data/src/core/ext/gcp/metadata_query.h +82 -0
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +70 -55
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +149 -60
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +5 -2
- data/src/core/ext/transport/chttp2/transport/flow_control.h +2 -1
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +4 -1
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +42 -23
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +5 -3
- data/src/core/ext/transport/chttp2/transport/internal.h +18 -3
- data/src/core/ext/transport/chttp2/transport/parsing.cc +9 -2
- data/src/core/ext/transport/chttp2/transport/writing.cc +10 -5
- data/src/core/ext/transport/inproc/inproc_transport.cc +20 -14
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +5 -3
- data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +22 -0
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +5 -3
- data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +22 -0
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +23 -5
- data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +94 -3
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +23 -2
- data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +120 -0
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +6 -3
- data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +22 -0
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +24 -6
- data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +111 -12
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +9 -7
- data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +27 -9
- data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c +0 -1
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +11 -7
- data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +56 -12
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c +5 -3
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +24 -0
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c +5 -3
- data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +24 -0
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +13 -2
- data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +49 -0
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +24 -9
- data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +66 -12
- data/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +191 -187
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c +139 -136
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.c +31 -15
- data/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c +12 -9
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.h +15 -0
- data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c +54 -45
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c +135 -119
- data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.h +5 -0
- data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +100 -97
- data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c +15 -18
- data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +272 -264
- data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +117 -117
- data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/ads.upbdefs.c +5 -5
- data/src/core/ext/upbdefs-generated/envoy/service/load_stats/v3/lrs.upbdefs.c +5 -5
- data/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c +5 -5
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.c +12 -9
- data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.h +5 -0
- data/src/core/ext/xds/xds_channel_stack_modifier.cc +1 -2
- data/src/core/ext/xds/xds_client_stats.cc +29 -15
- data/src/core/ext/xds/xds_client_stats.h +24 -20
- data/src/core/ext/xds/xds_endpoint.cc +5 -2
- data/src/core/ext/xds/xds_endpoint.h +9 -1
- data/src/core/ext/xds/xds_http_rbac_filter.cc +1 -1
- data/src/core/ext/xds/xds_lb_policy_registry.cc +13 -0
- data/src/core/ext/xds/xds_transport_grpc.cc +1 -1
- data/src/core/{ext/filters/client_channel/resolver/dns/dns_resolver_selection.h → lib/backoff/random_early_detection.cc} +14 -12
- data/src/core/lib/backoff/random_early_detection.h +59 -0
- data/src/core/lib/channel/call_finalization.h +1 -1
- data/src/core/lib/channel/call_tracer.cc +51 -0
- data/src/core/lib/channel/call_tracer.h +101 -38
- data/src/core/lib/channel/connected_channel.cc +483 -1050
- data/src/core/lib/channel/context.h +8 -1
- data/src/core/lib/channel/promise_based_filter.cc +106 -42
- data/src/core/lib/channel/promise_based_filter.h +27 -13
- data/src/core/lib/channel/server_call_tracer_filter.cc +110 -0
- data/src/core/lib/config/config_vars.cc +151 -0
- data/src/core/lib/config/config_vars.h +127 -0
- data/src/core/lib/config/config_vars_non_generated.cc +51 -0
- data/src/core/lib/config/load_config.cc +66 -0
- data/src/core/lib/config/load_config.h +49 -0
- data/src/core/lib/debug/trace.cc +5 -6
- data/src/core/lib/debug/trace.h +0 -5
- data/src/core/lib/event_engine/event_engine.cc +37 -2
- data/src/core/lib/event_engine/handle_containers.h +7 -22
- data/src/core/lib/event_engine/memory_allocator_factory.h +47 -0
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +0 -4
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +3 -9
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +48 -15
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +8 -8
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +6 -5
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +6 -3
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +27 -18
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +0 -3
- data/src/core/lib/event_engine/resolved_address.cc +2 -1
- data/src/core/lib/event_engine/windows/win_socket.cc +0 -1
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +129 -82
- data/src/core/lib/event_engine/windows/windows_endpoint.h +21 -5
- data/src/core/lib/event_engine/windows/windows_engine.cc +39 -18
- data/src/core/lib/event_engine/windows/windows_engine.h +2 -1
- data/src/core/lib/event_engine/windows/windows_listener.cc +370 -0
- data/src/core/lib/event_engine/windows/windows_listener.h +155 -0
- data/src/core/lib/experiments/config.cc +3 -10
- data/src/core/lib/experiments/experiments.cc +7 -0
- data/src/core/lib/experiments/experiments.h +9 -1
- data/src/core/lib/gpr/log.cc +15 -28
- data/src/core/lib/gprpp/fork.cc +8 -14
- data/src/core/lib/gprpp/orphanable.h +4 -3
- data/src/core/lib/gprpp/per_cpu.h +9 -3
- data/src/core/lib/gprpp/{thd_posix.cc → posix/thd.cc} +49 -37
- data/src/core/lib/gprpp/ref_counted.h +33 -34
- data/src/core/lib/gprpp/thd.h +16 -0
- data/src/core/lib/gprpp/time.cc +1 -0
- data/src/core/lib/gprpp/time.h +4 -4
- data/src/core/lib/gprpp/{thd_windows.cc → windows/thd.cc} +2 -2
- data/src/core/lib/iomgr/call_combiner.h +2 -2
- data/src/core/lib/iomgr/endpoint_cfstream.cc +4 -2
- data/src/core/lib/iomgr/endpoint_pair.h +2 -2
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -2
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +1 -1
- data/src/core/lib/iomgr/ev_posix.cc +13 -53
- data/src/core/lib/iomgr/ev_posix.h +0 -3
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +103 -76
- data/src/core/lib/iomgr/iomgr.cc +4 -8
- data/src/core/lib/iomgr/iomgr_windows.cc +8 -2
- data/src/core/lib/iomgr/pollset_set_windows.cc +9 -9
- data/src/core/lib/iomgr/pollset_windows.cc +1 -1
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +16 -3
- data/src/core/lib/iomgr/tcp_client_windows.cc +2 -2
- data/src/core/lib/iomgr/tcp_posix.cc +0 -1
- data/src/core/lib/iomgr/tcp_server_posix.cc +5 -16
- data/src/core/lib/iomgr/tcp_server_windows.cc +176 -9
- data/src/core/lib/iomgr/tcp_windows.cc +12 -8
- data/src/core/lib/load_balancing/lb_policy.cc +9 -13
- data/src/core/lib/load_balancing/lb_policy.h +4 -2
- data/src/core/lib/promise/activity.cc +22 -6
- data/src/core/lib/promise/activity.h +61 -24
- data/src/core/lib/promise/cancel_callback.h +77 -0
- data/src/core/lib/promise/detail/basic_seq.h +1 -1
- data/src/core/lib/promise/detail/promise_factory.h +4 -0
- data/src/core/lib/promise/for_each.h +176 -0
- data/src/core/lib/promise/if.h +9 -0
- data/src/core/lib/promise/interceptor_list.h +23 -2
- data/src/core/lib/promise/latch.h +89 -3
- data/src/core/lib/promise/loop.h +13 -9
- data/src/core/lib/promise/map.h +7 -0
- data/src/core/lib/promise/party.cc +286 -0
- data/src/core/lib/promise/party.h +499 -0
- data/src/core/lib/promise/pipe.h +197 -57
- data/src/core/lib/promise/poll.h +48 -0
- data/src/core/lib/promise/promise.h +2 -2
- data/src/core/lib/resource_quota/arena.cc +19 -3
- data/src/core/lib/resource_quota/arena.h +119 -5
- data/src/core/lib/resource_quota/memory_quota.cc +1 -1
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +12 -35
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +1 -0
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +0 -59
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +10 -5
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +13 -0
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +2 -0
- data/src/core/lib/security/security_connector/load_system_roots_supported.cc +5 -9
- data/src/core/lib/security/security_connector/ssl_utils.cc +11 -25
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +12 -0
- data/src/core/lib/security/transport/secure_endpoint.cc +4 -2
- data/src/core/lib/security/transport/server_auth_filter.cc +20 -2
- data/src/core/lib/slice/slice.cc +1 -1
- data/src/core/lib/surface/builtins.cc +2 -0
- data/src/core/lib/surface/call.cc +926 -1024
- data/src/core/lib/surface/call.h +10 -0
- data/src/core/lib/surface/lame_client.cc +1 -0
- data/src/core/lib/surface/version.cc +2 -2
- data/src/core/lib/transport/batch_builder.cc +179 -0
- data/src/core/lib/transport/batch_builder.h +468 -0
- data/src/core/lib/transport/bdp_estimator.cc +7 -7
- data/src/core/lib/transport/bdp_estimator.h +10 -6
- data/src/core/lib/transport/custom_metadata.h +30 -0
- data/src/core/lib/transport/metadata_batch.cc +9 -6
- data/src/core/lib/transport/metadata_batch.h +58 -16
- data/src/core/lib/transport/parsed_metadata.h +3 -3
- data/src/core/lib/transport/timeout_encoding.cc +6 -1
- data/src/core/lib/transport/transport.cc +30 -2
- data/src/core/lib/transport/transport.h +70 -14
- data/src/core/lib/transport/transport_impl.h +7 -0
- data/src/core/lib/transport/transport_op_string.cc +52 -42
- data/src/core/plugin_registry/grpc_plugin_registry.cc +2 -2
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +1 -0
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +21 -4
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +5 -0
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +1 -1
- data/src/core/tsi/ssl_transport_security.cc +4 -2
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/third_party/abseil-cpp/absl/base/config.h +1 -1
- data/third_party/abseil-cpp/absl/flags/commandlineflag.cc +34 -0
- data/third_party/abseil-cpp/absl/flags/commandlineflag.h +200 -0
- data/third_party/abseil-cpp/absl/flags/config.h +68 -0
- data/third_party/abseil-cpp/absl/flags/declare.h +73 -0
- data/third_party/abseil-cpp/absl/flags/flag.cc +38 -0
- data/third_party/abseil-cpp/absl/flags/flag.h +310 -0
- data/{src/core/lib/gprpp/global_config_custom.h → third_party/abseil-cpp/absl/flags/internal/commandlineflag.cc} +11 -14
- data/third_party/abseil-cpp/absl/flags/internal/commandlineflag.h +68 -0
- data/third_party/abseil-cpp/absl/flags/internal/flag.cc +615 -0
- data/third_party/abseil-cpp/absl/flags/internal/flag.h +800 -0
- data/third_party/abseil-cpp/absl/flags/internal/flag_msvc.inc +116 -0
- data/third_party/abseil-cpp/absl/flags/internal/path_util.h +62 -0
- data/third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc +65 -0
- data/third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.h +61 -0
- data/third_party/abseil-cpp/absl/flags/internal/program_name.cc +60 -0
- data/third_party/abseil-cpp/absl/flags/internal/program_name.h +50 -0
- data/third_party/abseil-cpp/absl/flags/internal/registry.h +97 -0
- data/third_party/abseil-cpp/absl/flags/internal/sequence_lock.h +187 -0
- data/third_party/abseil-cpp/absl/flags/marshalling.cc +241 -0
- data/third_party/abseil-cpp/absl/flags/marshalling.h +356 -0
- data/third_party/abseil-cpp/absl/flags/reflection.cc +354 -0
- data/third_party/abseil-cpp/absl/flags/reflection.h +90 -0
- data/third_party/abseil-cpp/absl/flags/usage_config.cc +165 -0
- data/third_party/abseil-cpp/absl/flags/usage_config.h +135 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +12 -8
- data/third_party/boringssl-with-bazel/err_data.c +728 -712
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_bitstr.c +177 -177
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_bool.c +28 -55
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_d2i_fp.c +21 -23
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_dup.c +20 -23
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_gentm.c +66 -185
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_i2d_fp.c +18 -21
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_int.c +356 -311
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_mbstr.c +174 -194
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_object.c +146 -210
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_octet.c +6 -9
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_strex.c +346 -526
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_strnid.c +110 -131
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_time.c +130 -116
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_type.c +93 -60
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_utctm.c +93 -181
- data/third_party/boringssl-with-bazel/src/crypto/asn1/asn1_lib.c +242 -305
- data/third_party/boringssl-with-bazel/src/crypto/asn1/asn1_par.c +41 -18
- data/third_party/boringssl-with-bazel/src/crypto/asn1/asn_pack.c +30 -33
- data/third_party/boringssl-with-bazel/src/crypto/asn1/f_int.c +36 -33
- data/third_party/boringssl-with-bazel/src/crypto/asn1/f_string.c +29 -26
- data/third_party/boringssl-with-bazel/src/crypto/asn1/internal.h +133 -88
- data/third_party/boringssl-with-bazel/src/crypto/asn1/posix_time.c +230 -0
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_dec.c +791 -791
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_enc.c +526 -526
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_fre.c +114 -135
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_new.c +201 -207
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_typ.c +21 -26
- data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_utl.c +55 -68
- data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +2 -4
- data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +11 -7
- data/third_party/boringssl-with-bazel/src/crypto/bio/bio_mem.c +4 -4
- data/third_party/boringssl-with-bazel/src/crypto/bio/connect.c +15 -9
- data/third_party/boringssl-with-bazel/src/crypto/bio/fd.c +4 -4
- data/third_party/boringssl-with-bazel/src/crypto/bio/file.c +17 -10
- data/third_party/boringssl-with-bazel/src/crypto/bio/pair.c +1 -3
- data/third_party/boringssl-with-bazel/src/crypto/bio/printf.c +0 -13
- data/third_party/boringssl-with-bazel/src/crypto/bio/socket.c +3 -6
- data/third_party/boringssl-with-bazel/src/crypto/bio/socket_helper.c +2 -0
- data/third_party/boringssl-with-bazel/src/crypto/blake2/blake2.c +9 -5
- data/third_party/boringssl-with-bazel/src/crypto/bn_extra/convert.c +10 -23
- data/third_party/boringssl-with-bazel/src/crypto/buf/buf.c +2 -6
- data/third_party/boringssl-with-bazel/src/crypto/bytestring/asn1_compat.c +2 -1
- data/third_party/boringssl-with-bazel/src/crypto/bytestring/ber.c +29 -28
- data/third_party/boringssl-with-bazel/src/crypto/bytestring/cbb.c +161 -201
- data/third_party/boringssl-with-bazel/src/crypto/bytestring/cbs.c +254 -39
- data/third_party/boringssl-with-bazel/src/crypto/bytestring/internal.h +2 -2
- data/third_party/boringssl-with-bazel/src/crypto/chacha/chacha.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/derive_key.c +4 -4
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesctrhmac.c +9 -8
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesgcmsiv.c +37 -75
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_chacha20poly1305.c +8 -10
- data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/cipher → cipher_extra}/e_des.c +100 -78
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_null.c +1 -0
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc2.c +1 -0
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc4.c +2 -0
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_tls.c +6 -12
- data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/internal.h +14 -11
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +6 -10
- data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/conf/internal.h +12 -0
- data/third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_apple.c +74 -0
- data/third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_freebsd.c +62 -0
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-fuchsia.c → cpu_aarch64_fuchsia.c} +8 -7
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-linux.c → cpu_aarch64_linux.c} +6 -4
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-win.c → cpu_aarch64_win.c} +4 -4
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm.c → cpu_arm.c} +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/cpu_arm_freebsd.c +55 -0
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm-linux.c → cpu_arm_linux.c} +11 -90
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm-linux.h → cpu_arm_linux.h} +0 -38
- data/third_party/boringssl-with-bazel/src/crypto/{cpu-intel.c → cpu_intel.c} +1 -2
- data/third_party/boringssl-with-bazel/src/crypto/crypto.c +25 -20
- data/third_party/boringssl-with-bazel/src/crypto/curve25519/curve25519.c +16 -27
- data/third_party/boringssl-with-bazel/src/crypto/curve25519/spake25519.c +17 -32
- data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/des → des}/des.c +232 -232
- data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/des → des}/internal.h +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/dh_extra/dh_asn1.c +1 -0
- data/third_party/boringssl-with-bazel/src/crypto/dh_extra/params.c +232 -29
- data/third_party/boringssl-with-bazel/src/crypto/digest_extra/digest_extra.c +0 -3
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +39 -16
- data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +37 -7
- data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +3 -3
- data/third_party/boringssl-with-bazel/src/crypto/ec_extra/ec_asn1.c +11 -36
- data/third_party/boringssl-with-bazel/src/crypto/ec_extra/hash_to_curve.c +214 -99
- data/third_party/boringssl-with-bazel/src/crypto/ec_extra/internal.h +21 -5
- data/third_party/boringssl-with-bazel/src/crypto/ecdsa_extra/ecdsa_asn1.c +2 -4
- data/third_party/boringssl-with-bazel/src/crypto/err/err.c +83 -60
- data/third_party/boringssl-with-bazel/src/crypto/evp/evp.c +46 -12
- data/third_party/boringssl-with-bazel/src/crypto/evp/evp_asn1.c +3 -3
- data/third_party/boringssl-with-bazel/src/crypto/evp/evp_ctx.c +25 -23
- data/third_party/boringssl-with-bazel/src/crypto/evp/internal.h +43 -9
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_dsa_asn1.c +75 -44
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec.c +19 -25
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec_asn1.c +96 -45
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519.c +7 -8
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519_asn1.c +26 -23
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_hkdf.c +233 -0
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_rsa.c +5 -5
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_rsa_asn1.c +42 -25
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_x25519.c +4 -5
- data/third_party/boringssl-with-bazel/src/crypto/evp/p_x25519_asn1.c +35 -47
- data/third_party/boringssl-with-bazel/src/crypto/evp/print.c +135 -244
- data/third_party/boringssl-with-bazel/src/crypto/evp/scrypt.c +2 -4
- data/third_party/boringssl-with-bazel/src/crypto/evp/sign.c +15 -10
- data/third_party/boringssl-with-bazel/src/crypto/ex_data.c +29 -15
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes_nohw.c +13 -14
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/internal.h +3 -13
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/key_wrap.c +13 -7
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/mode_wrappers.c +9 -7
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +35 -27
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bn.c +16 -26
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bytes.c +88 -60
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/cmp.c +4 -3
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/ctx.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div.c +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div_extra.c +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +99 -113
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd_extra.c +5 -3
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/generic.c +112 -168
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +86 -31
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery.c +11 -6
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery_inv.c +4 -5
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/mul.c +4 -5
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/prime.c +13 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/random.c +13 -5
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/rsaz_exp.c +19 -108
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/rsaz_exp.h +19 -15
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/shift.c +15 -16
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/sqrt.c +22 -21
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/aead.c +3 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/cipher.c +79 -19
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/e_aes.c +102 -99
- data/third_party/boringssl-with-bazel/src/crypto/{cipher_extra → fipsmodule/cipher}/e_aesccm.c +52 -46
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/internal.h +39 -0
- data/third_party/boringssl-with-bazel/src/crypto/{cmac → fipsmodule/cmac}/cmac.c +55 -11
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/check.c +2 -3
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/dh.c +21 -6
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/internal.h +56 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/digest/digest.c +5 -3
- data/third_party/boringssl-with-bazel/src/crypto/{evp → fipsmodule/digestsign}/digestsign.c +51 -15
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec.c +25 -25
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_key.c +91 -17
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_montgomery.c +5 -5
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/internal.h +34 -12
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/oct.c +54 -23
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p224-64.c +44 -60
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64-table.h → p256-nistz-table.h} +1 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64.c → p256-nistz.c} +60 -53
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64.h → p256-nistz.h} +5 -13
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p256.c +48 -36
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/scalar.c +2 -8
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/simple.c +2 -7
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/simple_mul.c +2 -3
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/wnaf.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdh/ecdh.c +8 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/ecdsa.c +42 -14
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/internal.h +6 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/hmac/hmac.c +52 -24
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/cbc.c +9 -15
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/cfb.c +1 -4
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/ctr.c +2 -4
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/gcm.c +71 -43
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/internal.h +14 -16
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/ofb.c +1 -4
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/ctrdrbg.c +31 -13
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.c +16 -8
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.h +3 -2
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/getrandom_fillin.h +2 -2
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/internal.h +9 -38
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/rand.c +73 -59
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/urandom.c +11 -45
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/blinding.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/internal.h +22 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/padding.c +63 -52
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa.c +107 -62
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa_impl.c +58 -31
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/fips.c +41 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/self_check.c +523 -422
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/internal.h +89 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/service_indicator.c +334 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +3 -12
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1.c +2 -0
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha256.c +12 -8
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +14 -12
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/tls/kdf.c +19 -6
- data/third_party/boringssl-with-bazel/src/crypto/hpke/hpke.c +32 -14
- data/third_party/boringssl-with-bazel/src/crypto/hrss/hrss.c +65 -29
- data/third_party/boringssl-with-bazel/src/crypto/internal.h +373 -18
- data/third_party/boringssl-with-bazel/src/crypto/kyber/internal.h +61 -0
- data/third_party/boringssl-with-bazel/src/crypto/kyber/keccak.c +205 -0
- data/third_party/boringssl-with-bazel/src/crypto/lhash/internal.h +13 -1
- data/third_party/boringssl-with-bazel/src/crypto/mem.c +220 -13
- data/third_party/boringssl-with-bazel/src/crypto/obj/obj.c +19 -7
- data/third_party/boringssl-with-bazel/src/crypto/obj/obj_dat.h +13 -1
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_all.c +81 -90
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_info.c +150 -245
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_lib.c +629 -613
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_oth.c +17 -17
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pk8.c +142 -149
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pkey.c +99 -131
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_x509.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/pem/pem_xaux.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/pkcs7/pkcs7_x509.c +0 -1
- data/third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8.c +0 -3
- data/third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8_x509.c +36 -66
- data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305.c +31 -38
- data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_arm.c +2 -1
- data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_vec.c +18 -31
- data/third_party/boringssl-with-bazel/src/crypto/pool/internal.h +1 -0
- data/third_party/boringssl-with-bazel/src/crypto/pool/pool.c +8 -1
- data/third_party/boringssl-with-bazel/src/crypto/rand_extra/passive.c +129 -5
- data/third_party/boringssl-with-bazel/src/crypto/refcount_c11.c +0 -2
- data/third_party/boringssl-with-bazel/src/crypto/refcount_lock.c +3 -4
- data/third_party/boringssl-with-bazel/src/crypto/siphash/siphash.c +8 -11
- data/third_party/boringssl-with-bazel/src/crypto/stack/stack.c +61 -27
- data/third_party/boringssl-with-bazel/src/crypto/thread_pthread.c +10 -13
- data/third_party/boringssl-with-bazel/src/crypto/thread_win.c +10 -13
- data/third_party/boringssl-with-bazel/src/crypto/trust_token/internal.h +66 -34
- data/third_party/boringssl-with-bazel/src/crypto/trust_token/pmbtoken.c +190 -77
- data/third_party/boringssl-with-bazel/src/crypto/trust_token/trust_token.c +81 -284
- data/third_party/boringssl-with-bazel/src/crypto/trust_token/voprf.c +109 -42
- data/third_party/boringssl-with-bazel/src/crypto/x509/a_digest.c +22 -24
- data/third_party/boringssl-with-bazel/src/crypto/x509/a_sign.c +54 -55
- data/third_party/boringssl-with-bazel/src/crypto/x509/a_verify.c +32 -34
- data/third_party/boringssl-with-bazel/src/crypto/x509/algorithm.c +32 -16
- data/third_party/boringssl-with-bazel/src/crypto/x509/asn1_gen.c +465 -704
- data/third_party/boringssl-with-bazel/src/crypto/x509/by_dir.c +284 -331
- data/third_party/boringssl-with-bazel/src/crypto/x509/by_file.c +183 -178
- data/third_party/boringssl-with-bazel/src/crypto/x509/i2d_pr.c +11 -15
- data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +67 -50
- data/third_party/boringssl-with-bazel/src/crypto/x509/name_print.c +153 -150
- data/third_party/boringssl-with-bazel/src/crypto/x509/policy.c +786 -0
- data/third_party/boringssl-with-bazel/src/crypto/x509/rsa_pss.c +95 -102
- data/third_party/boringssl-with-bazel/src/crypto/x509/t_crl.c +72 -57
- data/third_party/boringssl-with-bazel/src/crypto/x509/t_req.c +12 -10
- data/third_party/boringssl-with-bazel/src/crypto/x509/t_x509.c +227 -252
- data/third_party/boringssl-with-bazel/src/crypto/x509/t_x509a.c +52 -47
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509.c +3 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_att.c +230 -224
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_cmp.c +161 -327
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_d2.c +37 -33
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_def.c +14 -31
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_ext.c +55 -85
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +534 -618
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_obj.c +129 -122
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_req.c +116 -182
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_set.c +132 -132
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_trs.c +181 -202
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_txt.c +64 -79
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_v3.c +175 -160
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +1865 -2050
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vpm.c +433 -462
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509cset.c +156 -163
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509name.c +267 -263
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509rset.c +40 -15
- data/third_party/boringssl-with-bazel/src/crypto/x509/x509spki.c +59 -63
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_algor.c +63 -67
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_all.c +114 -144
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_attrib.c +25 -26
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_crl.c +326 -415
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_exten.c +8 -7
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_info.c +30 -28
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_name.c +354 -370
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_pkey.c +37 -32
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_pubkey.c +116 -119
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_req.c +36 -26
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_sig.c +3 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_spki.c +10 -13
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_val.c +3 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_x509.c +419 -261
- data/third_party/boringssl-with-bazel/src/crypto/x509/x_x509a.c +113 -105
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/ext_dat.h +11 -15
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/internal.h +78 -170
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_akey.c +126 -131
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_akeya.c +3 -4
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_alt.c +465 -469
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_bcons.c +56 -54
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_bitst.c +46 -49
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_conf.c +309 -346
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_cpols.c +341 -365
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_crld.c +429 -393
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_enum.c +29 -24
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_extku.c +65 -59
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_genn.c +125 -121
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ia5.c +43 -42
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_info.c +122 -125
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_int.c +50 -20
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_lib.c +247 -253
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ncons.c +386 -389
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ocsp.c +45 -32
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pcons.c +57 -54
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pmaps.c +63 -67
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_prn.c +143 -136
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_purp.c +664 -707
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_skey.c +83 -75
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_utl.c +1062 -1146
- data/third_party/boringssl-with-bazel/src/include/openssl/aead.h +8 -4
- data/third_party/boringssl-with-bazel/src/include/openssl/arm_arch.h +28 -48
- data/third_party/boringssl-with-bazel/src/include/openssl/asn1.h +211 -187
- data/third_party/boringssl-with-bazel/src/include/openssl/asn1t.h +26 -78
- data/third_party/boringssl-with-bazel/src/include/openssl/base.h +19 -14
- data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +21 -2
- data/third_party/boringssl-with-bazel/src/include/openssl/bn.h +49 -17
- data/third_party/boringssl-with-bazel/src/include/openssl/bytestring.h +99 -29
- data/third_party/boringssl-with-bazel/src/include/openssl/cipher.h +49 -60
- data/third_party/boringssl-with-bazel/src/include/openssl/conf.h +2 -15
- data/third_party/boringssl-with-bazel/src/include/openssl/cpu.h +16 -200
- data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +34 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/ctrdrbg.h +82 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/dh.h +32 -30
- data/third_party/boringssl-with-bazel/src/include/openssl/digest.h +7 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/dsa.h +4 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/ec.h +48 -5
- data/third_party/boringssl-with-bazel/src/include/openssl/ec_key.h +37 -8
- data/third_party/boringssl-with-bazel/src/include/openssl/ecdsa.h +1 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/err.h +33 -5
- data/third_party/boringssl-with-bazel/src/include/openssl/evp.h +22 -30
- data/third_party/boringssl-with-bazel/src/include/openssl/ex_data.h +1 -1
- data/third_party/boringssl-with-bazel/src/include/openssl/hmac.h +7 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/hpke.h +41 -16
- data/third_party/boringssl-with-bazel/src/include/openssl/kdf.h +91 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/mem.h +74 -8
- data/third_party/boringssl-with-bazel/src/include/openssl/nid.h +13 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/opensslconf.h +1 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/pem.h +11 -15
- data/third_party/boringssl-with-bazel/src/include/openssl/pkcs8.h +8 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/rand.h +12 -1
- data/third_party/boringssl-with-bazel/src/include/openssl/rsa.h +7 -4
- data/third_party/boringssl-with-bazel/src/include/openssl/service_indicator.h +96 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/span.h +13 -21
- data/third_party/boringssl-with-bazel/src/include/openssl/ssl.h +139 -75
- data/third_party/boringssl-with-bazel/src/include/openssl/ssl3.h +1 -6
- data/third_party/boringssl-with-bazel/src/include/openssl/stack.h +384 -286
- data/third_party/boringssl-with-bazel/src/include/openssl/thread.h +5 -6
- data/third_party/boringssl-with-bazel/src/include/openssl/time.h +41 -0
- data/third_party/boringssl-with-bazel/src/include/openssl/tls1.h +18 -7
- data/third_party/boringssl-with-bazel/src/include/openssl/trust_token.h +49 -23
- data/third_party/boringssl-with-bazel/src/include/openssl/type_check.h +0 -11
- data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +1592 -1074
- data/third_party/boringssl-with-bazel/src/include/openssl/x509v3.h +202 -205
- data/third_party/boringssl-with-bazel/src/ssl/bio_ssl.cc +2 -2
- data/third_party/boringssl-with-bazel/src/ssl/d1_both.cc +6 -13
- data/third_party/boringssl-with-bazel/src/ssl/d1_pkt.cc +17 -18
- data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +4 -5
- data/third_party/boringssl-with-bazel/src/ssl/dtls_record.cc +25 -33
- data/third_party/boringssl-with-bazel/src/ssl/encrypted_client_hello.cc +34 -20
- data/third_party/boringssl-with-bazel/src/ssl/extensions.cc +65 -34
- data/third_party/boringssl-with-bazel/src/ssl/handoff.cc +198 -54
- data/third_party/boringssl-with-bazel/src/ssl/handshake.cc +5 -5
- data/third_party/boringssl-with-bazel/src/ssl/handshake_client.cc +32 -28
- data/third_party/boringssl-with-bazel/src/ssl/handshake_server.cc +76 -44
- data/third_party/boringssl-with-bazel/src/ssl/internal.h +130 -98
- data/third_party/boringssl-with-bazel/src/ssl/s3_both.cc +27 -11
- data/third_party/boringssl-with-bazel/src/ssl/s3_lib.cc +2 -2
- data/third_party/boringssl-with-bazel/src/ssl/s3_pkt.cc +91 -75
- data/third_party/boringssl-with-bazel/src/ssl/ssl_aead_ctx.cc +8 -10
- data/third_party/boringssl-with-bazel/src/ssl/ssl_asn1.cc +39 -65
- data/third_party/boringssl-with-bazel/src/ssl/ssl_buffer.cc +1 -0
- data/third_party/boringssl-with-bazel/src/ssl/ssl_cert.cc +5 -9
- data/third_party/boringssl-with-bazel/src/ssl/ssl_cipher.cc +30 -33
- data/third_party/boringssl-with-bazel/src/ssl/ssl_file.cc +77 -100
- data/third_party/boringssl-with-bazel/src/ssl/ssl_key_share.cc +120 -107
- data/third_party/boringssl-with-bazel/src/ssl/ssl_lib.cc +164 -30
- data/third_party/boringssl-with-bazel/src/ssl/ssl_privkey.cc +150 -60
- data/third_party/boringssl-with-bazel/src/ssl/ssl_session.cc +22 -11
- data/third_party/boringssl-with-bazel/src/ssl/ssl_x509.cc +22 -6
- data/third_party/boringssl-with-bazel/src/ssl/t1_enc.cc +15 -13
- data/third_party/boringssl-with-bazel/src/ssl/tls13_both.cc +5 -43
- data/third_party/boringssl-with-bazel/src/ssl/tls13_client.cc +7 -4
- data/third_party/boringssl-with-bazel/src/ssl/tls13_enc.cc +2 -2
- data/third_party/boringssl-with-bazel/src/ssl/tls13_server.cc +22 -34
- data/third_party/boringssl-with-bazel/src/ssl/tls_method.cc +2 -2
- data/third_party/boringssl-with-bazel/src/ssl/tls_record.cc +16 -98
- data/third_party/boringssl-with-bazel/src/third_party/fiat/curve25519_32.h +1241 -657
- data/third_party/boringssl-with-bazel/src/third_party/fiat/curve25519_64.h +751 -398
- data/third_party/boringssl-with-bazel/src/third_party/fiat/p256_32.h +3551 -1938
- data/third_party/boringssl-with-bazel/src/third_party/fiat/p256_64.h +1272 -487
- metadata +107 -72
- data/src/core/ext/filters/client_channel/lb_call_state_internal.h +0 -39
- data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +0 -30
- data/src/core/lib/gprpp/global_config.h +0 -93
- data/src/core/lib/gprpp/global_config_env.cc +0 -140
- data/src/core/lib/gprpp/global_config_env.h +0 -133
- data/src/core/lib/gprpp/global_config_generic.h +0 -40
- data/src/core/lib/promise/intra_activity_waiter.h +0 -55
- data/src/core/lib/security/security_connector/ssl_utils_config.cc +0 -32
- data/src/core/lib/security/security_connector/ssl_utils_config.h +0 -29
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_enum.c +0 -195
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_print.c +0 -83
- data/third_party/boringssl-with-bazel/src/crypto/asn1/a_utf8.c +0 -236
- data/third_party/boringssl-with-bazel/src/crypto/asn1/charmap.h +0 -15
- data/third_party/boringssl-with-bazel/src/crypto/asn1/time_support.c +0 -206
- data/third_party/boringssl-with-bazel/src/crypto/cpu-ppc64le.c +0 -38
- data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1-altivec.c +0 -361
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_cache.c +0 -287
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_data.c +0 -132
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_lib.c +0 -155
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_map.c +0 -131
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_node.c +0 -189
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_tree.c +0 -843
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pci.c +0 -289
- data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pcia.c +0 -57
- /data/src/core/lib/gpr/{log_android.cc → android/log.cc} +0 -0
- /data/src/core/lib/gpr/{cpu_iphone.cc → iphone/cpu.cc} +0 -0
- /data/src/core/lib/gpr/{cpu_linux.cc → linux/cpu.cc} +0 -0
- /data/src/core/lib/gpr/{log_linux.cc → linux/log.cc} +0 -0
- /data/src/core/lib/gpr/{tmpfile_msys.cc → msys/tmpfile.cc} +0 -0
- /data/src/core/lib/gpr/{cpu_posix.cc → posix/cpu.cc} +0 -0
- /data/src/core/lib/gpr/{log_posix.cc → posix/log.cc} +0 -0
- /data/src/core/lib/gpr/{string_posix.cc → posix/string.cc} +0 -0
- /data/src/core/lib/gpr/{sync_posix.cc → posix/sync.cc} +0 -0
- /data/src/core/lib/gpr/{time_posix.cc → posix/time.cc} +0 -0
- /data/src/core/lib/gpr/{tmpfile_posix.cc → posix/tmpfile.cc} +0 -0
- /data/src/core/lib/gpr/{cpu_windows.cc → windows/cpu.cc} +0 -0
- /data/src/core/lib/gpr/{log_windows.cc → windows/log.cc} +0 -0
- /data/src/core/lib/gpr/{string_windows.cc → windows/string.cc} +0 -0
- /data/src/core/lib/gpr/{string_util_windows.cc → windows/string_util.cc} +0 -0
- /data/src/core/lib/gpr/{sync_windows.cc → windows/sync.cc} +0 -0
- /data/src/core/lib/gpr/{time_windows.cc → windows/time.cc} +0 -0
- /data/src/core/lib/gpr/{tmpfile_windows.cc → windows/tmpfile.cc} +0 -0
- /data/src/core/lib/gprpp/{env_linux.cc → linux/env.cc} +0 -0
- /data/src/core/lib/gprpp/{env_posix.cc → posix/env.cc} +0 -0
- /data/src/core/lib/gprpp/{stat_posix.cc → posix/stat.cc} +0 -0
- /data/src/core/lib/gprpp/{env_windows.cc → windows/env.cc} +0 -0
- /data/src/core/lib/gprpp/{stat_windows.cc → windows/stat.cc} +0 -0
@@ -23,6 +23,7 @@
|
|
23
23
|
#include <inttypes.h>
|
24
24
|
#include <limits.h>
|
25
25
|
#include <stdlib.h>
|
26
|
+
#include <string.h>
|
26
27
|
|
27
28
|
#include <algorithm>
|
28
29
|
#include <atomic>
|
@@ -35,14 +36,12 @@
|
|
35
36
|
#include <vector>
|
36
37
|
|
37
38
|
#include "absl/base/thread_annotations.h"
|
38
|
-
#include "absl/cleanup/cleanup.h"
|
39
39
|
#include "absl/meta/type_traits.h"
|
40
40
|
#include "absl/status/status.h"
|
41
41
|
#include "absl/strings/str_cat.h"
|
42
42
|
#include "absl/strings/str_format.h"
|
43
43
|
#include "absl/strings/str_join.h"
|
44
44
|
#include "absl/strings/string_view.h"
|
45
|
-
#include "absl/types/variant.h"
|
46
45
|
|
47
46
|
#include <grpc/byte_buffer.h>
|
48
47
|
#include <grpc/compression.h>
|
@@ -59,6 +58,8 @@
|
|
59
58
|
#include <grpc/support/time.h>
|
60
59
|
|
61
60
|
#include "src/core/lib/channel/call_finalization.h"
|
61
|
+
#include "src/core/lib/channel/call_tracer.h"
|
62
|
+
#include "src/core/lib/channel/channel_args.h"
|
62
63
|
#include "src/core/lib/channel/channel_stack.h"
|
63
64
|
#include "src/core/lib/channel/channelz.h"
|
64
65
|
#include "src/core/lib/channel/context.h"
|
@@ -85,8 +86,13 @@
|
|
85
86
|
#include "src/core/lib/promise/arena_promise.h"
|
86
87
|
#include "src/core/lib/promise/context.h"
|
87
88
|
#include "src/core/lib/promise/detail/basic_seq.h"
|
89
|
+
#include "src/core/lib/promise/latch.h"
|
90
|
+
#include "src/core/lib/promise/map.h"
|
91
|
+
#include "src/core/lib/promise/party.h"
|
88
92
|
#include "src/core/lib/promise/pipe.h"
|
89
93
|
#include "src/core/lib/promise/poll.h"
|
94
|
+
#include "src/core/lib/promise/race.h"
|
95
|
+
#include "src/core/lib/promise/seq.h"
|
90
96
|
#include "src/core/lib/resource_quota/arena.h"
|
91
97
|
#include "src/core/lib/slice/slice_buffer.h"
|
92
98
|
#include "src/core/lib/slice/slice_internal.h"
|
@@ -96,6 +102,7 @@
|
|
96
102
|
#include "src/core/lib/surface/completion_queue.h"
|
97
103
|
#include "src/core/lib/surface/server.h"
|
98
104
|
#include "src/core/lib/surface/validate_metadata.h"
|
105
|
+
#include "src/core/lib/transport/batch_builder.h"
|
99
106
|
#include "src/core/lib/transport/error_utils.h"
|
100
107
|
#include "src/core/lib/transport/metadata_batch.h"
|
101
108
|
#include "src/core/lib/transport/transport.h"
|
@@ -134,18 +141,18 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
134
141
|
virtual void InternalRef(const char* reason) = 0;
|
135
142
|
virtual void InternalUnref(const char* reason) = 0;
|
136
143
|
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
144
|
+
grpc_compression_algorithm test_only_compression_algorithm() {
|
145
|
+
return incoming_compression_algorithm_;
|
146
|
+
}
|
147
|
+
uint32_t test_only_message_flags() { return test_only_last_message_flags_; }
|
148
|
+
CompressionAlgorithmSet encodings_accepted_by_peer() {
|
149
|
+
return encodings_accepted_by_peer_;
|
150
|
+
}
|
142
151
|
|
143
152
|
// This should return nullptr for the promise stack (and alternative means
|
144
153
|
// for that functionality be invented)
|
145
154
|
virtual grpc_call_stack* call_stack() = 0;
|
146
155
|
|
147
|
-
gpr_atm* peer_string_atm_ptr() { return &peer_string_; }
|
148
|
-
|
149
156
|
protected:
|
150
157
|
// The maximum number of concurrent batches possible.
|
151
158
|
// Based upon the maximum number of individually queueable ops in the batch
|
@@ -203,7 +210,37 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
203
210
|
send_deadline_ = send_deadline;
|
204
211
|
}
|
205
212
|
|
206
|
-
|
213
|
+
Slice GetPeerString() const {
|
214
|
+
MutexLock lock(&peer_mu_);
|
215
|
+
return peer_string_.Ref();
|
216
|
+
}
|
217
|
+
|
218
|
+
void SetPeerString(Slice peer_string) {
|
219
|
+
MutexLock lock(&peer_mu_);
|
220
|
+
peer_string_ = std::move(peer_string);
|
221
|
+
}
|
222
|
+
|
223
|
+
void ClearPeerString() { SetPeerString(Slice(grpc_empty_slice())); }
|
224
|
+
|
225
|
+
// TODO(ctiller): cancel_func is for cancellation of the call - filter stack
|
226
|
+
// holds no mutexes here, promise stack does, and so locking is different.
|
227
|
+
// Remove this and cancel directly once promise conversion is done.
|
228
|
+
void ProcessIncomingInitialMetadata(grpc_metadata_batch& md);
|
229
|
+
// Fixup outgoing metadata before sending - adds compression, protects
|
230
|
+
// internal headers against external modification.
|
231
|
+
void PrepareOutgoingInitialMetadata(const grpc_op& op,
|
232
|
+
grpc_metadata_batch& md);
|
233
|
+
void NoteLastMessageFlags(uint32_t flags) {
|
234
|
+
test_only_last_message_flags_ = flags;
|
235
|
+
}
|
236
|
+
grpc_compression_algorithm incoming_compression_algorithm() const {
|
237
|
+
return incoming_compression_algorithm_;
|
238
|
+
}
|
239
|
+
|
240
|
+
void HandleCompressionAlgorithmDisabled(
|
241
|
+
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
242
|
+
void HandleCompressionAlgorithmNotAccepted(
|
243
|
+
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
207
244
|
|
208
245
|
private:
|
209
246
|
RefCountedPtr<Channel> channel_;
|
@@ -214,8 +251,18 @@ class Call : public CppImplOf<Call, grpc_call> {
|
|
214
251
|
const bool is_client_;
|
215
252
|
// flag indicating that cancellation is inherited
|
216
253
|
bool cancellation_is_inherited_ = false;
|
217
|
-
//
|
218
|
-
|
254
|
+
// Compression algorithm for *incoming* data
|
255
|
+
grpc_compression_algorithm incoming_compression_algorithm_ =
|
256
|
+
GRPC_COMPRESS_NONE;
|
257
|
+
// Supported encodings (compression algorithms), a bitset.
|
258
|
+
// Always support no compression.
|
259
|
+
CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
|
260
|
+
uint32_t test_only_last_message_flags_ = 0;
|
261
|
+
// Peer name is protected by a mutex because it can be accessed by the
|
262
|
+
// application at the same moment as it is being set by the completion
|
263
|
+
// of the recv_initial_metadata op. The mutex should be mostly uncontended.
|
264
|
+
mutable Mutex peer_mu_;
|
265
|
+
Slice peer_string_;
|
219
266
|
};
|
220
267
|
|
221
268
|
Call::ParentCall* Call::GetOrCreateParentCall() {
|
@@ -310,9 +357,13 @@ void Call::MaybeUnpublishFromParent() {
|
|
310
357
|
void Call::CancelWithStatus(grpc_status_code status, const char* description) {
|
311
358
|
// copying 'description' is needed to ensure the grpc_call_cancel_with_status
|
312
359
|
// guarantee that can be short-lived.
|
360
|
+
// TODO(ctiller): change to
|
361
|
+
// absl::Status(static_cast<absl::StatusCode>(status), description)
|
362
|
+
// (ie remove the set_int, set_str).
|
313
363
|
CancelWithError(grpc_error_set_int(
|
314
|
-
grpc_error_set_str(
|
315
|
-
|
364
|
+
grpc_error_set_str(
|
365
|
+
absl::Status(static_cast<absl::StatusCode>(status), description),
|
366
|
+
StatusStrProperty::kGrpcMessage, description),
|
316
367
|
StatusIntProperty::kRpcStatus, status));
|
317
368
|
}
|
318
369
|
|
@@ -337,9 +388,16 @@ void Call::PropagateCancellationToChildren() {
|
|
337
388
|
}
|
338
389
|
|
339
390
|
char* Call::GetPeer() {
|
340
|
-
|
341
|
-
if (
|
342
|
-
|
391
|
+
Slice peer_slice = GetPeerString();
|
392
|
+
if (!peer_slice.empty()) {
|
393
|
+
absl::string_view peer_string_view = peer_slice.as_string_view();
|
394
|
+
char* peer_string =
|
395
|
+
static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
|
396
|
+
memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
|
397
|
+
peer_string[peer_string_view.size()] = '\0';
|
398
|
+
return peer_string;
|
399
|
+
}
|
400
|
+
char* peer_string = grpc_channel_get_target(channel_->c_ptr());
|
343
401
|
if (peer_string != nullptr) return peer_string;
|
344
402
|
return gpr_strdup("unknown");
|
345
403
|
}
|
@@ -352,6 +410,92 @@ void Call::DeleteThis() {
|
|
352
410
|
arena->Destroy();
|
353
411
|
}
|
354
412
|
|
413
|
+
void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
|
414
|
+
grpc_metadata_batch& md) {
|
415
|
+
// TODO(juanlishen): If the user has already specified a compression
|
416
|
+
// algorithm by setting the initial metadata with key of
|
417
|
+
// GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
|
418
|
+
// with the compression algorithm mapped from compression level.
|
419
|
+
// process compression level
|
420
|
+
grpc_compression_level effective_compression_level = GRPC_COMPRESS_LEVEL_NONE;
|
421
|
+
bool level_set = false;
|
422
|
+
if (op.data.send_initial_metadata.maybe_compression_level.is_set) {
|
423
|
+
effective_compression_level =
|
424
|
+
op.data.send_initial_metadata.maybe_compression_level.level;
|
425
|
+
level_set = true;
|
426
|
+
} else {
|
427
|
+
const grpc_compression_options copts = channel()->compression_options();
|
428
|
+
if (copts.default_level.is_set) {
|
429
|
+
level_set = true;
|
430
|
+
effective_compression_level = copts.default_level.level;
|
431
|
+
}
|
432
|
+
}
|
433
|
+
// Currently, only server side supports compression level setting.
|
434
|
+
if (level_set && !is_client()) {
|
435
|
+
const grpc_compression_algorithm calgo =
|
436
|
+
encodings_accepted_by_peer().CompressionAlgorithmForLevel(
|
437
|
+
effective_compression_level);
|
438
|
+
// The following metadata will be checked and removed by the message
|
439
|
+
// compression filter. It will be used as the call's compression
|
440
|
+
// algorithm.
|
441
|
+
md.Set(GrpcInternalEncodingRequest(), calgo);
|
442
|
+
}
|
443
|
+
// Ignore any te metadata key value pairs specified.
|
444
|
+
md.Remove(TeMetadata());
|
445
|
+
}
|
446
|
+
|
447
|
+
void Call::ProcessIncomingInitialMetadata(grpc_metadata_batch& md) {
|
448
|
+
Slice* peer_string = md.get_pointer(PeerString());
|
449
|
+
if (peer_string != nullptr) SetPeerString(peer_string->Ref());
|
450
|
+
|
451
|
+
incoming_compression_algorithm_ =
|
452
|
+
md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
453
|
+
encodings_accepted_by_peer_ =
|
454
|
+
md.Take(GrpcAcceptEncodingMetadata())
|
455
|
+
.value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
|
456
|
+
|
457
|
+
const grpc_compression_options compression_options =
|
458
|
+
channel_->compression_options();
|
459
|
+
const grpc_compression_algorithm compression_algorithm =
|
460
|
+
incoming_compression_algorithm_;
|
461
|
+
if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
|
462
|
+
compression_options.enabled_algorithms_bitset)
|
463
|
+
.IsSet(compression_algorithm))) {
|
464
|
+
// check if algorithm is supported by current channel config
|
465
|
+
HandleCompressionAlgorithmDisabled(compression_algorithm);
|
466
|
+
}
|
467
|
+
// GRPC_COMPRESS_NONE is always set.
|
468
|
+
GPR_DEBUG_ASSERT(encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
469
|
+
if (GPR_UNLIKELY(!encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
470
|
+
if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
|
471
|
+
HandleCompressionAlgorithmNotAccepted(compression_algorithm);
|
472
|
+
}
|
473
|
+
}
|
474
|
+
}
|
475
|
+
|
476
|
+
void Call::HandleCompressionAlgorithmNotAccepted(
|
477
|
+
grpc_compression_algorithm compression_algorithm) {
|
478
|
+
const char* algo_name = nullptr;
|
479
|
+
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
480
|
+
gpr_log(GPR_ERROR,
|
481
|
+
"Compression algorithm ('%s') not present in the "
|
482
|
+
"accepted encodings (%s)",
|
483
|
+
algo_name,
|
484
|
+
std::string(encodings_accepted_by_peer_.ToString()).c_str());
|
485
|
+
}
|
486
|
+
|
487
|
+
void Call::HandleCompressionAlgorithmDisabled(
|
488
|
+
grpc_compression_algorithm compression_algorithm) {
|
489
|
+
const char* algo_name = nullptr;
|
490
|
+
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
491
|
+
std::string error_msg =
|
492
|
+
absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
|
493
|
+
gpr_log(GPR_ERROR, "%s", error_msg.c_str());
|
494
|
+
CancelWithError(grpc_error_set_int(absl::UnimplementedError(error_msg),
|
495
|
+
StatusIntProperty::kRpcStatus,
|
496
|
+
GRPC_STATUS_UNIMPLEMENTED));
|
497
|
+
}
|
498
|
+
|
355
499
|
///////////////////////////////////////////////////////////////////////////////
|
356
500
|
// FilterStackCall
|
357
501
|
// To be removed once promise conversion is complete
|
@@ -410,11 +554,6 @@ class FilterStackCall final : public Call {
|
|
410
554
|
return context_[elem].value;
|
411
555
|
}
|
412
556
|
|
413
|
-
grpc_compression_algorithm compression_for_level(
|
414
|
-
grpc_compression_level level) override {
|
415
|
-
return encodings_accepted_by_peer_.CompressionAlgorithmForLevel(level);
|
416
|
-
}
|
417
|
-
|
418
557
|
bool is_trailers_only() const override {
|
419
558
|
bool result = is_trailers_only_;
|
420
559
|
GPR_DEBUG_ASSERT(!result || recv_initial_metadata_.TransportSize() == 0);
|
@@ -432,18 +571,6 @@ class FilterStackCall final : public Call {
|
|
432
571
|
return authority_metadata->as_string_view();
|
433
572
|
}
|
434
573
|
|
435
|
-
grpc_compression_algorithm test_only_compression_algorithm() override {
|
436
|
-
return incoming_compression_algorithm_;
|
437
|
-
}
|
438
|
-
|
439
|
-
uint32_t test_only_message_flags() override {
|
440
|
-
return test_only_last_message_flags_;
|
441
|
-
}
|
442
|
-
|
443
|
-
uint32_t test_only_encodings_accepted_by_peer() override {
|
444
|
-
return encodings_accepted_by_peer_.ToLegacyBitmask();
|
445
|
-
}
|
446
|
-
|
447
574
|
static size_t InitialSizeEstimate() {
|
448
575
|
return sizeof(FilterStackCall) +
|
449
576
|
sizeof(BatchControl) * kMaxConcurrentBatches;
|
@@ -480,6 +607,7 @@ class FilterStackCall final : public Call {
|
|
480
607
|
}
|
481
608
|
struct BatchControl {
|
482
609
|
FilterStackCall* call_ = nullptr;
|
610
|
+
CallTracerAnnotationInterface* call_tracer_ = nullptr;
|
483
611
|
grpc_transport_stream_op_batch op_;
|
484
612
|
// Share memory for cq_completion and notify_tag as they are never needed
|
485
613
|
// simultaneously. Each byte used in this data structure count as six bytes
|
@@ -509,12 +637,31 @@ class FilterStackCall final : public Call {
|
|
509
637
|
}
|
510
638
|
bool completed_batch_step(PendingOp op) {
|
511
639
|
auto mask = PendingOpMask(op);
|
640
|
+
// Acquire call tracer before ops_pending_.fetch_sub to avoid races with
|
641
|
+
// call_ being set to nullptr in PostCompletion method. Store the
|
642
|
+
// call_tracer_ and call_ variables locally as well because they could be
|
643
|
+
// modified by another thread after the fetch_sub operation.
|
644
|
+
CallTracerAnnotationInterface* call_tracer = call_tracer_;
|
645
|
+
FilterStackCall* call = call_;
|
646
|
+
bool is_call_trace_enabled = grpc_call_trace.enabled();
|
647
|
+
bool is_call_ops_annotate_enabled =
|
648
|
+
(IsTraceRecordCallopsEnabled() && call_tracer != nullptr);
|
649
|
+
if (is_call_ops_annotate_enabled) {
|
650
|
+
call->InternalRef("Call ops annotate");
|
651
|
+
}
|
512
652
|
auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
|
513
|
-
if (
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
653
|
+
if (is_call_trace_enabled || is_call_ops_annotate_enabled) {
|
654
|
+
std::string trace_string = absl::StrFormat(
|
655
|
+
"BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
|
656
|
+
PendingOpString(mask).c_str(), PendingOpString(r & ~mask).c_str(),
|
657
|
+
completion_data_.notify_tag.tag);
|
658
|
+
if (is_call_trace_enabled) {
|
659
|
+
gpr_log(GPR_DEBUG, "%s", trace_string.c_str());
|
660
|
+
}
|
661
|
+
if (is_call_ops_annotate_enabled) {
|
662
|
+
call_tracer->RecordAnnotation(trace_string);
|
663
|
+
call->InternalUnref("Call ops annotate");
|
664
|
+
}
|
518
665
|
}
|
519
666
|
GPR_ASSERT((r & mask) != 0);
|
520
667
|
return r == mask;
|
@@ -524,7 +671,6 @@ class FilterStackCall final : public Call {
|
|
524
671
|
void FinishStep(PendingOp op);
|
525
672
|
void ProcessDataAfterMetadata();
|
526
673
|
void ReceivingStreamReady(grpc_error_handle error);
|
527
|
-
void ValidateFilteredMetadata();
|
528
674
|
void ReceivingInitialMetadataReady(grpc_error_handle error);
|
529
675
|
void ReceivingTrailingMetadataReady(grpc_error_handle error);
|
530
676
|
void FinishBatch(grpc_error_handle error);
|
@@ -549,10 +695,6 @@ class FilterStackCall final : public Call {
|
|
549
695
|
grpc_closure* start_batch_closure);
|
550
696
|
void SetFinalStatus(grpc_error_handle error);
|
551
697
|
BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
|
552
|
-
void HandleCompressionAlgorithmDisabled(
|
553
|
-
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
554
|
-
void HandleCompressionAlgorithmNotAccepted(
|
555
|
-
grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
|
556
698
|
bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
|
557
699
|
bool is_trailing);
|
558
700
|
void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
|
@@ -596,13 +738,6 @@ class FilterStackCall final : public Call {
|
|
596
738
|
// completed
|
597
739
|
grpc_call_final_info final_info_;
|
598
740
|
|
599
|
-
// Compression algorithm for *incoming* data
|
600
|
-
grpc_compression_algorithm incoming_compression_algorithm_ =
|
601
|
-
GRPC_COMPRESS_NONE;
|
602
|
-
// Supported encodings (compression algorithms), a bitset.
|
603
|
-
// Always support no compression.
|
604
|
-
CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
|
605
|
-
|
606
741
|
// Contexts for various subsystems (security, tracing, ...).
|
607
742
|
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
608
743
|
|
@@ -616,7 +751,6 @@ class FilterStackCall final : public Call {
|
|
616
751
|
grpc_closure receiving_stream_ready_;
|
617
752
|
grpc_closure receiving_initial_metadata_ready_;
|
618
753
|
grpc_closure receiving_trailing_metadata_ready_;
|
619
|
-
uint32_t test_only_last_message_flags_ = 0;
|
620
754
|
// Status about operation of call
|
621
755
|
bool sent_server_trailing_metadata_ = false;
|
622
756
|
gpr_atm cancelled_with_error_ = 0;
|
@@ -703,6 +837,26 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
|
|
703
837
|
global_stats().IncrementServerCallsCreated();
|
704
838
|
call->final_op_.server.cancelled = nullptr;
|
705
839
|
call->final_op_.server.core_server = args->server;
|
840
|
+
// TODO(yashykt): In the future, we want to also enable stats and trace
|
841
|
+
// collecting from when the call is created at the transport. The idea is
|
842
|
+
// that the transport would create the call tracer and pass it in as part of
|
843
|
+
// the metadata.
|
844
|
+
auto* server_call_tracer_factory = ServerCallTracerFactory::Get(
|
845
|
+
args->server != nullptr ? args->server->channel_args() : ChannelArgs());
|
846
|
+
if (server_call_tracer_factory != nullptr) {
|
847
|
+
auto* server_call_tracer =
|
848
|
+
server_call_tracer_factory->CreateNewServerCallTracer(arena);
|
849
|
+
if (server_call_tracer != nullptr) {
|
850
|
+
// Note that we are setting both
|
851
|
+
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
852
|
+
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
853
|
+
// promise-based world, we would just a single tracer object for each
|
854
|
+
// stack (call, subchannel_call, server_call.)
|
855
|
+
call->ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
856
|
+
server_call_tracer, nullptr);
|
857
|
+
call->ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
858
|
+
}
|
859
|
+
}
|
706
860
|
}
|
707
861
|
|
708
862
|
Call* parent = Call::FromC(args->parent);
|
@@ -1033,11 +1187,7 @@ void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
|
|
1033
1187
|
}
|
1034
1188
|
|
1035
1189
|
void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
|
1036
|
-
|
1037
|
-
b->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
|
1038
|
-
encodings_accepted_by_peer_ =
|
1039
|
-
b->Take(GrpcAcceptEncodingMetadata())
|
1040
|
-
.value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
|
1190
|
+
ProcessIncomingInitialMetadata(*b);
|
1041
1191
|
PublishAppMetadata(b, false);
|
1042
1192
|
}
|
1043
1193
|
|
@@ -1052,11 +1202,11 @@ void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
|
|
1052
1202
|
grpc_status_code status_code = *grpc_status;
|
1053
1203
|
grpc_error_handle error;
|
1054
1204
|
if (status_code != GRPC_STATUS_OK) {
|
1055
|
-
|
1205
|
+
Slice peer = GetPeerString();
|
1056
1206
|
error = grpc_error_set_int(
|
1057
|
-
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
|
1207
|
+
GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
|
1208
|
+
peer.as_string_view())),
|
1058
1209
|
StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
|
1059
|
-
gpr_free(peer);
|
1060
1210
|
}
|
1061
1211
|
auto grpc_message = b->Take(GrpcMessageMetadata());
|
1062
1212
|
if (grpc_message.has_value()) {
|
@@ -1133,6 +1283,8 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
|
|
1133
1283
|
*pslot = bctl;
|
1134
1284
|
}
|
1135
1285
|
bctl->call_ = this;
|
1286
|
+
bctl->call_tracer_ = static_cast<CallTracerAnnotationInterface*>(
|
1287
|
+
ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
|
1136
1288
|
bctl->op_.payload = &stream_op_payload_;
|
1137
1289
|
return bctl;
|
1138
1290
|
}
|
@@ -1204,11 +1356,11 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
|
|
1204
1356
|
call->receiving_message_ = false;
|
1205
1357
|
FinishStep(PendingOp::kRecvMessage);
|
1206
1358
|
} else {
|
1207
|
-
call->
|
1359
|
+
call->NoteLastMessageFlags(call->receiving_stream_flags_);
|
1208
1360
|
if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
1209
|
-
(call->
|
1361
|
+
(call->incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
1210
1362
|
*call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
|
1211
|
-
nullptr, 0, call->
|
1363
|
+
nullptr, 0, call->incoming_compression_algorithm());
|
1212
1364
|
} else {
|
1213
1365
|
*call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
1214
1366
|
}
|
@@ -1249,50 +1401,6 @@ void FilterStackCall::BatchControl::ReceivingStreamReady(
|
|
1249
1401
|
}
|
1250
1402
|
}
|
1251
1403
|
|
1252
|
-
void FilterStackCall::HandleCompressionAlgorithmDisabled(
|
1253
|
-
grpc_compression_algorithm compression_algorithm) {
|
1254
|
-
const char* algo_name = nullptr;
|
1255
|
-
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
1256
|
-
std::string error_msg =
|
1257
|
-
absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
|
1258
|
-
gpr_log(GPR_ERROR, "%s", error_msg.c_str());
|
1259
|
-
CancelWithStatus(GRPC_STATUS_UNIMPLEMENTED, error_msg.c_str());
|
1260
|
-
}
|
1261
|
-
|
1262
|
-
void FilterStackCall::HandleCompressionAlgorithmNotAccepted(
|
1263
|
-
grpc_compression_algorithm compression_algorithm) {
|
1264
|
-
const char* algo_name = nullptr;
|
1265
|
-
grpc_compression_algorithm_name(compression_algorithm, &algo_name);
|
1266
|
-
gpr_log(GPR_ERROR,
|
1267
|
-
"Compression algorithm ('%s') not present in the "
|
1268
|
-
"accepted encodings (%s)",
|
1269
|
-
algo_name,
|
1270
|
-
std::string(encodings_accepted_by_peer_.ToString()).c_str());
|
1271
|
-
}
|
1272
|
-
|
1273
|
-
void FilterStackCall::BatchControl::ValidateFilteredMetadata() {
|
1274
|
-
FilterStackCall* call = call_;
|
1275
|
-
|
1276
|
-
const grpc_compression_options compression_options =
|
1277
|
-
call->channel()->compression_options();
|
1278
|
-
const grpc_compression_algorithm compression_algorithm =
|
1279
|
-
call->incoming_compression_algorithm_;
|
1280
|
-
if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
|
1281
|
-
compression_options.enabled_algorithms_bitset)
|
1282
|
-
.IsSet(compression_algorithm))) {
|
1283
|
-
// check if algorithm is supported by current channel config
|
1284
|
-
call->HandleCompressionAlgorithmDisabled(compression_algorithm);
|
1285
|
-
}
|
1286
|
-
// GRPC_COMPRESS_NONE is always set.
|
1287
|
-
GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
|
1288
|
-
if (GPR_UNLIKELY(
|
1289
|
-
!call->encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
|
1290
|
-
if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
|
1291
|
-
call->HandleCompressionAlgorithmNotAccepted(compression_algorithm);
|
1292
|
-
}
|
1293
|
-
}
|
1294
|
-
}
|
1295
|
-
|
1296
1404
|
void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
1297
1405
|
grpc_error_handle error) {
|
1298
1406
|
FilterStackCall* call = call_;
|
@@ -1303,9 +1411,6 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
|
|
1303
1411
|
grpc_metadata_batch* md = &call->recv_initial_metadata_;
|
1304
1412
|
call->RecvInitialFilter(md);
|
1305
1413
|
|
1306
|
-
// TODO(ctiller): this could be moved into recv_initial_filter now
|
1307
|
-
ValidateFilteredMetadata();
|
1308
|
-
|
1309
1414
|
absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
|
1310
1415
|
if (deadline.has_value() && !call->is_client()) {
|
1311
1416
|
call_->set_send_deadline(*deadline);
|
@@ -1400,6 +1505,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1400
1505
|
grpc_transport_stream_op_batch_payload* stream_op_payload;
|
1401
1506
|
uint32_t seen_ops = 0;
|
1402
1507
|
intptr_t pending_ops = 0;
|
1508
|
+
CallTracerAnnotationInterface* call_tracer = nullptr;
|
1403
1509
|
|
1404
1510
|
for (i = 0; i < nops; i++) {
|
1405
1511
|
if (seen_ops & (1u << ops[i].op)) {
|
@@ -1454,36 +1560,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1454
1560
|
error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
|
1455
1561
|
goto done_with_error;
|
1456
1562
|
}
|
1457
|
-
// TODO(juanlishen): If the user has already specified a compression
|
1458
|
-
// algorithm by setting the initial metadata with key of
|
1459
|
-
// GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
|
1460
|
-
// with the compression algorithm mapped from compression level.
|
1461
|
-
// process compression level
|
1462
|
-
grpc_compression_level effective_compression_level =
|
1463
|
-
GRPC_COMPRESS_LEVEL_NONE;
|
1464
|
-
bool level_set = false;
|
1465
|
-
if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
|
1466
|
-
effective_compression_level =
|
1467
|
-
op->data.send_initial_metadata.maybe_compression_level.level;
|
1468
|
-
level_set = true;
|
1469
|
-
} else {
|
1470
|
-
const grpc_compression_options copts =
|
1471
|
-
channel()->compression_options();
|
1472
|
-
if (copts.default_level.is_set) {
|
1473
|
-
level_set = true;
|
1474
|
-
effective_compression_level = copts.default_level.level;
|
1475
|
-
}
|
1476
|
-
}
|
1477
|
-
// Currently, only server side supports compression level setting.
|
1478
|
-
if (level_set && !is_client()) {
|
1479
|
-
const grpc_compression_algorithm calgo =
|
1480
|
-
encodings_accepted_by_peer_.CompressionAlgorithmForLevel(
|
1481
|
-
effective_compression_level);
|
1482
|
-
// The following metadata will be checked and removed by the message
|
1483
|
-
// compression filter. It will be used as the call's compression
|
1484
|
-
// algorithm.
|
1485
|
-
send_initial_metadata_.Set(GrpcInternalEncodingRequest(), calgo);
|
1486
|
-
}
|
1487
1563
|
if (op->data.send_initial_metadata.count > INT_MAX) {
|
1488
1564
|
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1489
1565
|
goto done_with_error;
|
@@ -1496,8 +1572,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1496
1572
|
error = GRPC_CALL_ERROR_INVALID_METADATA;
|
1497
1573
|
goto done_with_error;
|
1498
1574
|
}
|
1499
|
-
|
1500
|
-
send_initial_metadata_.Remove(TeMetadata());
|
1575
|
+
PrepareOutgoingInitialMetadata(*op, send_initial_metadata_);
|
1501
1576
|
// TODO(ctiller): just make these the same variable?
|
1502
1577
|
if (is_client() && send_deadline() != Timestamp::InfFuture()) {
|
1503
1578
|
send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
|
@@ -1512,10 +1587,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1512
1587
|
}
|
1513
1588
|
stream_op_payload->send_initial_metadata.send_initial_metadata =
|
1514
1589
|
&send_initial_metadata_;
|
1515
|
-
if (is_client()) {
|
1516
|
-
stream_op_payload->send_initial_metadata.peer_string =
|
1517
|
-
peer_string_atm_ptr();
|
1518
|
-
}
|
1519
1590
|
pending_ops |= PendingOpMask(PendingOp::kSends);
|
1520
1591
|
break;
|
1521
1592
|
}
|
@@ -1664,9 +1735,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1664
1735
|
if (is_client()) {
|
1665
1736
|
stream_op_payload->recv_initial_metadata.trailing_metadata_available =
|
1666
1737
|
&is_trailers_only_;
|
1667
|
-
} else {
|
1668
|
-
stream_op_payload->recv_initial_metadata.peer_string =
|
1669
|
-
peer_string_atm_ptr();
|
1670
1738
|
}
|
1671
1739
|
pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
|
1672
1740
|
break;
|
@@ -1798,13 +1866,21 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
|
|
1798
1866
|
stream_op->on_complete = &bctl->finish_batch_;
|
1799
1867
|
}
|
1800
1868
|
|
1869
|
+
call_tracer = static_cast<CallTracerAnnotationInterface*>(
|
1870
|
+
ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
|
1871
|
+
if ((IsTraceRecordCallopsEnabled() && call_tracer != nullptr)) {
|
1872
|
+
call_tracer->RecordAnnotation(absl::StrFormat(
|
1873
|
+
"BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1874
|
+
PendingOpString(pending_ops).c_str(),
|
1875
|
+
grpc_transport_stream_op_batch_string(stream_op, true).c_str(),
|
1876
|
+
bctl->completion_data_.notify_tag.tag));
|
1877
|
+
}
|
1801
1878
|
if (grpc_call_trace.enabled()) {
|
1802
1879
|
gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
|
1803
1880
|
PendingOpString(pending_ops).c_str(),
|
1804
|
-
grpc_transport_stream_op_batch_string(stream_op).c_str(),
|
1881
|
+
grpc_transport_stream_op_batch_string(stream_op, false).c_str(),
|
1805
1882
|
bctl->completion_data_.notify_tag.tag);
|
1806
1883
|
}
|
1807
|
-
|
1808
1884
|
ExecuteBatch(stream_op, &bctl->start_batch_);
|
1809
1885
|
|
1810
1886
|
done:
|
@@ -1873,8 +1949,7 @@ bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
|
|
1873
1949
|
// Will be folded into Call once the promise conversion is done
|
1874
1950
|
|
1875
1951
|
class PromiseBasedCall : public Call,
|
1876
|
-
public
|
1877
|
-
public Wakeable,
|
1952
|
+
public Party,
|
1878
1953
|
public grpc_event_engine::experimental::EventEngine::
|
1879
1954
|
Closure /* for deadlines */ {
|
1880
1955
|
public:
|
@@ -1885,176 +1960,54 @@ class PromiseBasedCall : public Call,
|
|
1885
1960
|
void (*destroy)(void* value)) override;
|
1886
1961
|
void* ContextGet(grpc_context_index elem) const override;
|
1887
1962
|
void SetCompletionQueue(grpc_completion_queue* cq) override;
|
1888
|
-
|
1889
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
1890
|
-
void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) {
|
1891
|
-
MutexLock lock(&mu_);
|
1892
|
-
CancelWithErrorLocked(std::move(error));
|
1893
|
-
}
|
1894
|
-
virtual void CancelWithErrorLocked(absl::Status error)
|
1895
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
1896
|
-
bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) {
|
1897
|
-
MutexLock lock(&mu_);
|
1898
|
-
return completed_;
|
1899
|
-
}
|
1900
|
-
|
1901
|
-
void Orphan() final {
|
1902
|
-
MutexLock lock(&mu_);
|
1903
|
-
if (!completed_) {
|
1904
|
-
CancelWithErrorLocked(absl::CancelledError("Call orphaned"));
|
1905
|
-
}
|
1906
|
-
}
|
1963
|
+
bool Completed() final { return finished_.IsSet(); }
|
1907
1964
|
|
1908
1965
|
// Implementation of call refcounting: move this to DualRefCounted once we
|
1909
1966
|
// don't need to maintain FilterStackCall compatibility
|
1910
|
-
void ExternalRef() final {
|
1911
|
-
|
1912
|
-
refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed);
|
1913
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1914
|
-
gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(),
|
1915
|
-
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1916
|
-
GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair));
|
1917
|
-
}
|
1918
|
-
}
|
1919
|
-
void ExternalUnref() final {
|
1920
|
-
const uint64_t prev_ref_pair =
|
1921
|
-
refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel);
|
1922
|
-
if (grpc_call_refcount_trace.enabled()) {
|
1923
|
-
gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(),
|
1924
|
-
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1925
|
-
GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1);
|
1926
|
-
}
|
1927
|
-
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
1928
|
-
if (GPR_UNLIKELY(strong_refs == 1)) {
|
1929
|
-
Orphan();
|
1930
|
-
}
|
1931
|
-
// Now drop the weak ref.
|
1932
|
-
InternalUnref("external_ref");
|
1933
|
-
}
|
1967
|
+
void ExternalRef() final { InternalRef("external"); }
|
1968
|
+
void ExternalUnref() final { InternalUnref("external"); }
|
1934
1969
|
void InternalRef(const char* reason) final {
|
1935
|
-
uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed);
|
1936
1970
|
if (grpc_call_refcount_trace.enabled()) {
|
1937
|
-
gpr_log(GPR_DEBUG, "
|
1938
|
-
GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n),
|
1939
|
-
GetWeakRefs(n) + 1);
|
1971
|
+
gpr_log(GPR_DEBUG, "INTERNAL_REF:%p:%s", this, reason);
|
1940
1972
|
}
|
1973
|
+
Party::IncrementRefCount();
|
1941
1974
|
}
|
1942
1975
|
void InternalUnref(const char* reason) final {
|
1943
|
-
const uint64_t prev_ref_pair =
|
1944
|
-
refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel);
|
1945
1976
|
if (grpc_call_refcount_trace.enabled()) {
|
1946
|
-
gpr_log(GPR_DEBUG, "
|
1947
|
-
reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
|
1948
|
-
GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1);
|
1949
|
-
}
|
1950
|
-
if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
|
1951
|
-
DeleteThis();
|
1977
|
+
gpr_log(GPR_DEBUG, "INTERNAL_UNREF:%p:%s", this, reason);
|
1952
1978
|
}
|
1979
|
+
Party::Unref();
|
1953
1980
|
}
|
1954
1981
|
|
1955
|
-
// Activity methods
|
1956
|
-
void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1957
|
-
Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override {
|
1958
|
-
InternalRef("wakeup");
|
1959
|
-
// If ASAN is defined, we leverage it to detect dropped Waker objects.
|
1960
|
-
// Usually Waker must be destroyed or woken up, but (especially with arenas)
|
1961
|
-
// it's not uncommon to create a Waker and then do neither. In that case it's
|
1962
|
-
// incredibly fraught to diagnose where the dropped reference to this object was
|
1963
|
-
// created. Instead, leverage ASAN and create a new object per expected wakeup.
|
1964
|
-
// Now when we drop such an object ASAN will fail and we'll get a callstack to
|
1965
|
-
// the creation of the waker in question.
|
1966
|
-
#if defined(__has_feature)
|
1967
|
-
#if __has_feature(address_sanitizer)
|
1968
|
-
#define GRPC_CALL_USES_ASAN_WAKER
|
1969
|
-
class AsanWaker final : public Wakeable {
|
1970
|
-
public:
|
1971
|
-
explicit AsanWaker(PromiseBasedCall* call) : call_(call) {}
|
1972
|
-
|
1973
|
-
void Wakeup(void*) override {
|
1974
|
-
call_->Wakeup(nullptr);
|
1975
|
-
delete this;
|
1976
|
-
}
|
1977
|
-
|
1978
|
-
void Drop(void*) override {
|
1979
|
-
call_->Drop(nullptr);
|
1980
|
-
delete this;
|
1981
|
-
}
|
1982
|
-
|
1983
|
-
std::string ActivityDebugTag(void*) const override {
|
1984
|
-
return call_->DebugTag();
|
1985
|
-
}
|
1986
|
-
|
1987
|
-
private:
|
1988
|
-
PromiseBasedCall* call_;
|
1989
|
-
};
|
1990
|
-
return Waker(new AsanWaker(this), nullptr);
|
1991
|
-
#endif
|
1992
|
-
#endif
|
1993
|
-
#ifndef GRPC_CALL_USES_ASAN_WAKER
|
1994
|
-
return Waker(this, nullptr);
|
1995
|
-
#endif
|
1996
|
-
}
|
1997
|
-
Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
|
1998
|
-
|
1999
|
-
// Wakeable methods
|
2000
|
-
void Wakeup(void*) override {
|
2001
|
-
channel()->event_engine()->Run([this] {
|
2002
|
-
ApplicationCallbackExecCtx app_exec_ctx;
|
2003
|
-
ExecCtx exec_ctx;
|
2004
|
-
{
|
2005
|
-
ScopedContext activity_context(this);
|
2006
|
-
MutexLock lock(&mu_);
|
2007
|
-
Update();
|
2008
|
-
}
|
2009
|
-
InternalUnref("wakeup");
|
2010
|
-
});
|
2011
|
-
}
|
2012
|
-
void Drop(void*) override { InternalUnref("wakeup"); }
|
2013
|
-
|
2014
1982
|
void RunInContext(absl::AnyInvocable<void()> fn) {
|
2015
|
-
|
2016
|
-
|
2017
|
-
|
2018
|
-
InternalRef("in_context");
|
2019
|
-
channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable {
|
2020
|
-
ApplicationCallbackExecCtx app_exec_ctx;
|
2021
|
-
ExecCtx exec_ctx;
|
2022
|
-
{
|
2023
|
-
ScopedContext activity_context(this);
|
2024
|
-
MutexLock lock(&mu_);
|
1983
|
+
Spawn(
|
1984
|
+
"run_in_context",
|
1985
|
+
[fn = std::move(fn)]() mutable {
|
2025
1986
|
fn();
|
2026
|
-
|
2027
|
-
}
|
2028
|
-
|
2029
|
-
});
|
2030
|
-
}
|
2031
|
-
}
|
2032
|
-
|
2033
|
-
grpc_compression_algorithm test_only_compression_algorithm() override {
|
2034
|
-
abort();
|
2035
|
-
}
|
2036
|
-
uint32_t test_only_message_flags() override { abort(); }
|
2037
|
-
uint32_t test_only_encodings_accepted_by_peer() override { abort(); }
|
2038
|
-
grpc_compression_algorithm compression_for_level(
|
2039
|
-
grpc_compression_level) override {
|
2040
|
-
abort();
|
1987
|
+
return Empty{};
|
1988
|
+
},
|
1989
|
+
[](Empty) {});
|
2041
1990
|
}
|
2042
1991
|
|
2043
1992
|
// This should return nullptr for the promise stack (and alternative means
|
2044
1993
|
// for that functionality be invented)
|
2045
1994
|
grpc_call_stack* call_stack() override { return nullptr; }
|
2046
1995
|
|
2047
|
-
void UpdateDeadline(Timestamp deadline);
|
2048
|
-
void ResetDeadline();
|
1996
|
+
void UpdateDeadline(Timestamp deadline) ABSL_LOCKS_EXCLUDED(deadline_mu_);
|
1997
|
+
void ResetDeadline() ABSL_LOCKS_EXCLUDED(deadline_mu_);
|
2049
1998
|
|
2050
1999
|
// Implementation of EventEngine::Closure, called when deadline expires
|
2051
2000
|
void Run() override;
|
2052
2001
|
|
2053
2002
|
virtual ServerCallContext* server_call_context() { return nullptr; }
|
2054
2003
|
|
2004
|
+
using Call::arena;
|
2005
|
+
|
2055
2006
|
protected:
|
2056
2007
|
class ScopedContext
|
2057
2008
|
: public ScopedActivity,
|
2009
|
+
public BatchBuilder,
|
2010
|
+
public promise_detail::Context<BatchBuilder>,
|
2058
2011
|
public promise_detail::Context<Arena>,
|
2059
2012
|
public promise_detail::Context<grpc_call_context_element>,
|
2060
2013
|
public promise_detail::Context<CallContext>,
|
@@ -2062,6 +2015,8 @@ class PromiseBasedCall : public Call,
|
|
2062
2015
|
public:
|
2063
2016
|
explicit ScopedContext(PromiseBasedCall* call)
|
2064
2017
|
: ScopedActivity(call),
|
2018
|
+
BatchBuilder(&call->batch_payload_),
|
2019
|
+
promise_detail::Context<BatchBuilder>(this),
|
2065
2020
|
promise_detail::Context<Arena>(call->arena()),
|
2066
2021
|
promise_detail::Context<grpc_call_context_element>(call->context_),
|
2067
2022
|
promise_detail::Context<CallContext>(&call->call_context_),
|
@@ -2095,8 +2050,12 @@ class PromiseBasedCall : public Call,
|
|
2095
2050
|
};
|
2096
2051
|
|
2097
2052
|
~PromiseBasedCall() override {
|
2098
|
-
if (non_owning_wakeable_) non_owning_wakeable_->DropActivity();
|
2099
2053
|
if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
|
2054
|
+
for (int i = 0; i < GRPC_CONTEXT_COUNT; i++) {
|
2055
|
+
if (context_[i].destroy) {
|
2056
|
+
context_[i].destroy(context_[i].value);
|
2057
|
+
}
|
2058
|
+
}
|
2100
2059
|
}
|
2101
2060
|
|
2102
2061
|
// Enumerates why a Completion is still pending
|
@@ -2104,6 +2063,7 @@ class PromiseBasedCall : public Call,
|
|
2104
2063
|
// We're in the midst of starting a batch of operations
|
2105
2064
|
kStartingBatch = 0,
|
2106
2065
|
// The following correspond with the batch operations from above
|
2066
|
+
kSendInitialMetadata,
|
2107
2067
|
kReceiveInitialMetadata,
|
2108
2068
|
kReceiveStatusOnClient,
|
2109
2069
|
kReceiveCloseOnServer = kReceiveStatusOnClient,
|
@@ -2113,10 +2073,17 @@ class PromiseBasedCall : public Call,
|
|
2113
2073
|
kSendCloseFromClient = kSendStatusFromServer,
|
2114
2074
|
};
|
2115
2075
|
|
2076
|
+
bool RunParty() override {
|
2077
|
+
ScopedContext ctx(this);
|
2078
|
+
return Party::RunParty();
|
2079
|
+
}
|
2080
|
+
|
2116
2081
|
const char* PendingOpString(PendingOp reason) const {
|
2117
2082
|
switch (reason) {
|
2118
2083
|
case PendingOp::kStartingBatch:
|
2119
2084
|
return "StartingBatch";
|
2085
|
+
case PendingOp::kSendInitialMetadata:
|
2086
|
+
return "SendInitialMetadata";
|
2120
2087
|
case PendingOp::kReceiveInitialMetadata:
|
2121
2088
|
return "ReceiveInitialMetadata";
|
2122
2089
|
case PendingOp::kReceiveStatusOnClient:
|
@@ -2131,56 +2098,47 @@ class PromiseBasedCall : public Call,
|
|
2131
2098
|
return "Unknown";
|
2132
2099
|
}
|
2133
2100
|
|
2134
|
-
static constexpr
|
2101
|
+
static constexpr uint32_t PendingOpBit(PendingOp reason) {
|
2135
2102
|
return 1 << static_cast<int>(reason);
|
2136
2103
|
}
|
2137
2104
|
|
2138
|
-
Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; }
|
2139
2105
|
// Begin work on a completion, recording the tag/closure to notify.
|
2140
2106
|
// Use the op selected in \a ops to determine the index to allocate into.
|
2141
2107
|
// Starts the "StartingBatch" PendingOp immediately.
|
2142
2108
|
// Assumes at least one operation in \a ops.
|
2143
|
-
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops)
|
2144
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2109
|
+
Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops);
|
2145
2110
|
// Add one pending op to the completion, and return it.
|
2146
|
-
Completion AddOpToCompletion(const Completion& completion, PendingOp reason)
|
2147
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2111
|
+
Completion AddOpToCompletion(const Completion& completion, PendingOp reason);
|
2148
2112
|
// Stringify a completion
|
2149
2113
|
std::string CompletionString(const Completion& completion) const {
|
2150
2114
|
return completion.has_value()
|
2151
|
-
?
|
2152
|
-
"%d:tag=%p", static_cast<int>(completion.index()),
|
2153
|
-
completion_info_[completion.index()].pending.tag)
|
2115
|
+
? completion_info_[completion.index()].pending.ToString(this)
|
2154
2116
|
: "no-completion";
|
2155
2117
|
}
|
2156
2118
|
// Finish one op on the completion. Must have been previously been added.
|
2157
2119
|
// The completion as a whole finishes when all pending ops finish.
|
2158
|
-
void FinishOpOnCompletion(Completion* completion, PendingOp reason)
|
2159
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2120
|
+
void FinishOpOnCompletion(Completion* completion, PendingOp reason);
|
2160
2121
|
// Mark the completion as failed. Does not finish it.
|
2161
2122
|
void FailCompletion(const Completion& completion,
|
2162
2123
|
SourceLocation source_location = {});
|
2163
|
-
//
|
2164
|
-
|
2165
|
-
|
2166
|
-
virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
2124
|
+
// Mark the completion as infallible. Overrides FailCompletion to report
|
2125
|
+
// success always.
|
2126
|
+
void ForceCompletionSuccess(const Completion& completion);
|
2167
2127
|
// Accept the stats from the context (call once we have proof the transport is
|
2168
2128
|
// done with them).
|
2169
2129
|
// Right now this means that promise based calls do not record correct stats
|
2170
2130
|
// with census if they are cancelled.
|
2171
2131
|
// TODO(ctiller): this should be remedied before promise based calls are
|
2172
2132
|
// dexperimentalized.
|
2173
|
-
void AcceptTransportStatsFromContext()
|
2133
|
+
void AcceptTransportStatsFromContext() {
|
2174
2134
|
final_stats_ = *call_context_.call_stats();
|
2175
2135
|
}
|
2176
2136
|
|
2177
|
-
grpc_completion_queue* cq()
|
2137
|
+
grpc_completion_queue* cq() { return cq_; }
|
2178
2138
|
|
2179
2139
|
void CToMetadata(grpc_metadata* metadata, size_t count,
|
2180
2140
|
grpc_metadata_batch* batch);
|
2181
2141
|
|
2182
|
-
std::string ActivityDebugTag(void*) const override { return DebugTag(); }
|
2183
|
-
|
2184
2142
|
// At the end of the call run any finalization actions.
|
2185
2143
|
void RunFinalization(grpc_status_code status, const char* status_details) {
|
2186
2144
|
grpc_call_final_info final_info;
|
@@ -2209,158 +2167,163 @@ class PromiseBasedCall : public Call,
|
|
2209
2167
|
}
|
2210
2168
|
}
|
2211
2169
|
|
2212
|
-
|
2213
|
-
|
2214
|
-
outstanding_send_.has_value(),
|
2215
|
-
send_message_completion_)
|
2216
|
-
.c_str(),
|
2217
|
-
PresentAndCompletionText("outstanding_recv",
|
2218
|
-
outstanding_recv_.has_value(),
|
2219
|
-
recv_message_completion_)
|
2220
|
-
.c_str());
|
2221
|
-
}
|
2222
|
-
|
2170
|
+
// Spawn a job that will first do FirstPromise then receive a message
|
2171
|
+
template <typename FirstPromise>
|
2223
2172
|
void StartRecvMessage(const grpc_op& op, const Completion& completion,
|
2224
|
-
|
2225
|
-
|
2226
|
-
|
2227
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2228
|
-
void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
2173
|
+
FirstPromise first,
|
2174
|
+
PipeReceiver<MessageHandle>* receiver,
|
2175
|
+
Party::BulkSpawner& spawner);
|
2229
2176
|
void StartSendMessage(const grpc_op& op, const Completion& completion,
|
2230
|
-
PipeSender<MessageHandle>* sender
|
2231
|
-
|
2232
|
-
|
2233
|
-
void
|
2177
|
+
PipeSender<MessageHandle>* sender,
|
2178
|
+
Party::BulkSpawner& spawner);
|
2179
|
+
|
2180
|
+
void set_completed() { finished_.Set(); }
|
2234
2181
|
|
2235
|
-
|
2236
|
-
|
2182
|
+
// Returns a promise that resolves to Empty whenever the call is completed.
|
2183
|
+
auto finished() { return finished_.Wait(); }
|
2184
|
+
|
2185
|
+
// Returns a promise that resolves to Empty whenever there is no outstanding
|
2186
|
+
// send operation
|
2187
|
+
auto WaitForSendingStarted() {
|
2188
|
+
return [this]() -> Poll<Empty> {
|
2189
|
+
int n = sends_queued_.load(std::memory_order_relaxed);
|
2190
|
+
if (grpc_call_trace.enabled()) {
|
2191
|
+
gpr_log(GPR_DEBUG, "%s[call] WaitForSendingStarted n=%d",
|
2192
|
+
DebugTag().c_str(), n);
|
2193
|
+
}
|
2194
|
+
if (n != 0) return waiting_for_queued_sends_.pending();
|
2195
|
+
return Empty{};
|
2196
|
+
};
|
2237
2197
|
}
|
2238
|
-
|
2239
|
-
|
2240
|
-
|
2198
|
+
|
2199
|
+
// Mark that a send has been queued - blocks sending trailing metadata.
|
2200
|
+
void QueueSend() {
|
2201
|
+
if (grpc_call_trace.enabled()) {
|
2202
|
+
gpr_log(GPR_DEBUG, "%s[call] QueueSend", DebugTag().c_str());
|
2203
|
+
}
|
2204
|
+
sends_queued_.fetch_add(1, std::memory_order_relaxed);
|
2205
|
+
}
|
2206
|
+
// Mark that a send has been dequeued - allows sending trailing metadata once
|
2207
|
+
// zero sends are queued.
|
2208
|
+
void EnactSend() {
|
2209
|
+
if (grpc_call_trace.enabled()) {
|
2210
|
+
gpr_log(GPR_DEBUG, "%s[call] EnactSend", DebugTag().c_str());
|
2211
|
+
}
|
2212
|
+
if (1 == sends_queued_.fetch_sub(1, std::memory_order_relaxed)) {
|
2213
|
+
waiting_for_queued_sends_.Wake();
|
2214
|
+
}
|
2241
2215
|
}
|
2242
2216
|
|
2243
2217
|
private:
|
2244
2218
|
union CompletionInfo {
|
2219
|
+
static constexpr uint32_t kOpFailed = 0x8000'0000u;
|
2220
|
+
static constexpr uint32_t kOpForceSuccess = 0x4000'0000u;
|
2221
|
+
CompletionInfo() {}
|
2222
|
+
enum CompletionState {
|
2223
|
+
kPending,
|
2224
|
+
kSuccess,
|
2225
|
+
kFailure,
|
2226
|
+
};
|
2245
2227
|
struct Pending {
|
2246
|
-
// Bitmask of PendingOps
|
2247
|
-
|
2228
|
+
// Bitmask of PendingOps at the bottom, and kOpFailed, kOpForceSuccess at
|
2229
|
+
// the top.
|
2230
|
+
std::atomic<uint32_t> state;
|
2248
2231
|
bool is_closure;
|
2249
|
-
|
2232
|
+
// True if this completion was for a recv_message op.
|
2233
|
+
// In that case if the completion as a whole fails we need to cleanup the
|
2234
|
+
// returned message.
|
2235
|
+
bool is_recv_message;
|
2250
2236
|
void* tag;
|
2251
|
-
} pending;
|
2252
|
-
grpc_cq_completion completion;
|
2253
|
-
};
|
2254
2237
|
|
2255
|
-
|
2256
|
-
|
2257
|
-
|
2258
|
-
|
2259
|
-
|
2260
|
-
|
2261
|
-
|
2262
|
-
// Activity is going away... drop its reference and sever the connection
|
2263
|
-
// back.
|
2264
|
-
void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
|
2265
|
-
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2266
|
-
MutexLock lock(&mu_);
|
2267
|
-
GPR_ASSERT(call_ != nullptr);
|
2268
|
-
call_ = nullptr;
|
2269
|
-
}
|
2238
|
+
void Start(bool is_closure, void* tag) {
|
2239
|
+
this->is_closure = is_closure;
|
2240
|
+
this->is_recv_message = false;
|
2241
|
+
this->tag = tag;
|
2242
|
+
state.store(PendingOpBit(PendingOp::kStartingBatch),
|
2243
|
+
std::memory_order_release);
|
2244
|
+
}
|
2270
2245
|
|
2271
|
-
|
2272
|
-
|
2273
|
-
|
2274
|
-
|
2275
|
-
|
2276
|
-
auto unref = absl::MakeCleanup([this]() { Unref(); });
|
2277
|
-
ReleasableMutexLock lock(&mu_);
|
2278
|
-
// Note that activity refcount can drop to zero, but we could win the lock
|
2279
|
-
// against DropActivity, so we need to only increase activities refcount
|
2280
|
-
// if it is non-zero.
|
2281
|
-
PromiseBasedCall* call = call_;
|
2282
|
-
if (call != nullptr && call->RefIfNonZero()) {
|
2283
|
-
lock.Release();
|
2284
|
-
// Activity still exists and we have a reference: wake it up, which will
|
2285
|
-
// drop the ref.
|
2286
|
-
call->Wakeup(nullptr);
|
2246
|
+
void AddPendingBit(PendingOp reason) {
|
2247
|
+
if (reason == PendingOp::kReceiveMessage) is_recv_message = true;
|
2248
|
+
auto prev =
|
2249
|
+
state.fetch_or(PendingOpBit(reason), std::memory_order_relaxed);
|
2250
|
+
GPR_ASSERT((prev & PendingOpBit(reason)) == 0);
|
2287
2251
|
}
|
2288
|
-
}
|
2289
2252
|
|
2290
|
-
|
2291
|
-
|
2292
|
-
|
2293
|
-
|
2253
|
+
CompletionState RemovePendingBit(PendingOp reason) {
|
2254
|
+
const uint32_t mask = ~PendingOpBit(reason);
|
2255
|
+
auto prev = state.fetch_and(mask, std::memory_order_acq_rel);
|
2256
|
+
GPR_ASSERT((prev & PendingOpBit(reason)) != 0);
|
2257
|
+
switch (prev & mask) {
|
2258
|
+
case kOpFailed:
|
2259
|
+
return kFailure;
|
2260
|
+
case kOpFailed | kOpForceSuccess:
|
2261
|
+
case kOpForceSuccess:
|
2262
|
+
case 0:
|
2263
|
+
return kSuccess;
|
2264
|
+
default:
|
2265
|
+
return kPending;
|
2266
|
+
}
|
2267
|
+
}
|
2294
2268
|
|
2295
|
-
|
2269
|
+
void MarkFailed() {
|
2270
|
+
state.fetch_or(kOpFailed, std::memory_order_relaxed);
|
2271
|
+
}
|
2296
2272
|
|
2297
|
-
|
2298
|
-
|
2299
|
-
void Unref() {
|
2300
|
-
if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) {
|
2301
|
-
delete this;
|
2273
|
+
void MarkForceSuccess() {
|
2274
|
+
state.fetch_or(kOpForceSuccess, std::memory_order_relaxed);
|
2302
2275
|
}
|
2303
|
-
}
|
2304
2276
|
|
2305
|
-
|
2306
|
-
|
2307
|
-
|
2308
|
-
|
2309
|
-
|
2310
|
-
|
2277
|
+
std::string ToString(const PromiseBasedCall* call) const {
|
2278
|
+
auto state = this->state.load(std::memory_order_relaxed);
|
2279
|
+
std::vector<absl::string_view> pending_ops;
|
2280
|
+
for (size_t i = 0; i < 24; i++) {
|
2281
|
+
if (state & (1u << i)) {
|
2282
|
+
pending_ops.push_back(
|
2283
|
+
call->PendingOpString(static_cast<PendingOp>(i)));
|
2284
|
+
}
|
2285
|
+
}
|
2286
|
+
return absl::StrFormat("{%s}%s:tag=%p", absl::StrJoin(pending_ops, ","),
|
2287
|
+
(state & kOpForceSuccess) ? ":force-success"
|
2288
|
+
: (state & kOpFailed) ? ":failed"
|
2289
|
+
: ":success",
|
2290
|
+
tag);
|
2291
|
+
}
|
2292
|
+
} pending;
|
2293
|
+
grpc_cq_completion completion;
|
2311
2294
|
};
|
2312
2295
|
|
2313
|
-
|
2314
|
-
|
2315
|
-
|
2316
|
-
|
2317
|
-
|
2318
|
-
|
2319
|
-
|
2320
|
-
static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
|
2321
|
-
return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
|
2322
|
-
}
|
2323
|
-
static uint32_t GetStrongRefs(uint64_t ref_pair) {
|
2324
|
-
return static_cast<uint32_t>(ref_pair >> 32);
|
2325
|
-
}
|
2326
|
-
static uint32_t GetWeakRefs(uint64_t ref_pair) {
|
2327
|
-
return static_cast<uint32_t>(ref_pair & 0xffffffffu);
|
2328
|
-
}
|
2329
|
-
|
2330
|
-
bool RefIfNonZero() {
|
2331
|
-
uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire);
|
2332
|
-
do {
|
2333
|
-
const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
|
2334
|
-
if (strong_refs == 0) return false;
|
2335
|
-
} while (!refs_.compare_exchange_weak(
|
2336
|
-
prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0),
|
2337
|
-
std::memory_order_acq_rel, std::memory_order_acquire));
|
2338
|
-
return true;
|
2296
|
+
void PartyOver() override {
|
2297
|
+
{
|
2298
|
+
ScopedContext ctx(this);
|
2299
|
+
CancelRemainingParticipants();
|
2300
|
+
arena()->DestroyManagedNewObjects();
|
2301
|
+
}
|
2302
|
+
DeleteThis();
|
2339
2303
|
}
|
2340
2304
|
|
2341
|
-
mutable Mutex mu_;
|
2342
|
-
std::atomic<uint64_t> refs_;
|
2343
2305
|
CallContext call_context_{this};
|
2344
|
-
bool keep_polling_ ABSL_GUARDED_BY(mu()) = false;
|
2345
2306
|
|
2346
2307
|
// Contexts for various subsystems (security, tracing, ...).
|
2347
2308
|
grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
|
2348
|
-
grpc_completion_queue* cq_
|
2349
|
-
NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr;
|
2309
|
+
grpc_completion_queue* cq_;
|
2350
2310
|
CompletionInfo completion_info_[6];
|
2351
2311
|
grpc_call_stats final_stats_{};
|
2352
2312
|
CallFinalization finalization_;
|
2353
2313
|
// Current deadline.
|
2354
|
-
|
2355
|
-
|
2356
|
-
|
2357
|
-
|
2358
|
-
|
2359
|
-
|
2360
|
-
|
2361
|
-
|
2362
|
-
|
2363
|
-
|
2314
|
+
Mutex deadline_mu_;
|
2315
|
+
Timestamp deadline_ ABSL_GUARDED_BY(deadline_mu_) = Timestamp::InfFuture();
|
2316
|
+
grpc_event_engine::experimental::EventEngine::TaskHandle ABSL_GUARDED_BY(
|
2317
|
+
deadline_mu_) deadline_task_;
|
2318
|
+
ExternallyObservableLatch<void> finished_;
|
2319
|
+
// Non-zero with an outstanding GRPC_OP_SEND_INITIAL_METADATA or
|
2320
|
+
// GRPC_OP_SEND_MESSAGE (one count each), and 0 once those payloads have been
|
2321
|
+
// pushed onto the outgoing pipe.
|
2322
|
+
std::atomic<uint8_t> sends_queued_{0};
|
2323
|
+
// Waiter for when sends_queued_ becomes 0.
|
2324
|
+
IntraActivityWaiter waiting_for_queued_sends_;
|
2325
|
+
grpc_byte_buffer** recv_message_ = nullptr;
|
2326
|
+
grpc_transport_stream_op_batch_payload batch_payload_{context_};
|
2364
2327
|
};
|
2365
2328
|
|
2366
2329
|
template <typename T>
|
@@ -2380,7 +2343,7 @@ PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
|
2380
2343
|
const grpc_call_create_args& args)
|
2381
2344
|
: Call(arena, args.server_transport_data == nullptr, args.send_deadline,
|
2382
2345
|
args.channel->Ref()),
|
2383
|
-
|
2346
|
+
Party(arena, initial_external_refs),
|
2384
2347
|
cq_(args.cq) {
|
2385
2348
|
if (args.cq != nullptr) {
|
2386
2349
|
GPR_ASSERT(args.pollset_set_alternative == nullptr &&
|
@@ -2396,15 +2359,6 @@ PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
|
|
2396
2359
|
}
|
2397
2360
|
}
|
2398
2361
|
|
2399
|
-
Waker PromiseBasedCall::MakeNonOwningWaker() {
|
2400
|
-
if (non_owning_wakeable_ == nullptr) {
|
2401
|
-
non_owning_wakeable_ = new NonOwningWakable(this);
|
2402
|
-
} else {
|
2403
|
-
non_owning_wakeable_->Ref();
|
2404
|
-
}
|
2405
|
-
return Waker(non_owning_wakeable_, nullptr);
|
2406
|
-
}
|
2407
|
-
|
2408
2362
|
void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count,
|
2409
2363
|
grpc_metadata_batch* b) {
|
2410
2364
|
for (size_t i = 0; i < count; i++) {
|
@@ -2439,15 +2393,14 @@ void* PromiseBasedCall::ContextGet(grpc_context_index elem) const {
|
|
2439
2393
|
PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
|
2440
2394
|
void* tag, bool is_closure, const grpc_op* ops) {
|
2441
2395
|
Completion c(BatchSlotForOp(ops[0].op));
|
2442
|
-
if (grpc_call_trace.enabled()) {
|
2443
|
-
gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(),
|
2444
|
-
CompletionString(c).c_str(), tag);
|
2445
|
-
}
|
2446
2396
|
if (!is_closure) {
|
2447
2397
|
grpc_cq_begin_op(cq(), tag);
|
2448
2398
|
}
|
2449
|
-
completion_info_[c.index()].pending
|
2450
|
-
|
2399
|
+
completion_info_[c.index()].pending.Start(is_closure, tag);
|
2400
|
+
if (grpc_call_trace.enabled()) {
|
2401
|
+
gpr_log(GPR_INFO, "%s[call] StartCompletion %s", DebugTag().c_str(),
|
2402
|
+
CompletionString(c).c_str());
|
2403
|
+
}
|
2451
2404
|
return c;
|
2452
2405
|
}
|
2453
2406
|
|
@@ -2458,10 +2411,7 @@ PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
|
|
2458
2411
|
CompletionString(completion).c_str(), PendingOpString(reason));
|
2459
2412
|
}
|
2460
2413
|
GPR_ASSERT(completion.has_value());
|
2461
|
-
|
2462
|
-
completion_info_[completion.index()].pending.pending_op_bits;
|
2463
|
-
GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0);
|
2464
|
-
pending_op_bits |= PendingOpBit(reason);
|
2414
|
+
completion_info_[completion.index()].pending.AddPendingBit(reason);
|
2465
2415
|
return Completion(completion.index());
|
2466
2416
|
}
|
2467
2417
|
|
@@ -2472,64 +2422,50 @@ void PromiseBasedCall::FailCompletion(const Completion& completion,
|
|
2472
2422
|
"%s[call] FailCompletion %s", DebugTag().c_str(),
|
2473
2423
|
CompletionString(completion).c_str());
|
2474
2424
|
}
|
2475
|
-
completion_info_[completion.index()].pending.
|
2425
|
+
completion_info_[completion.index()].pending.MarkFailed();
|
2426
|
+
}
|
2427
|
+
|
2428
|
+
void PromiseBasedCall::ForceCompletionSuccess(const Completion& completion) {
|
2429
|
+
completion_info_[completion.index()].pending.MarkForceSuccess();
|
2476
2430
|
}
|
2477
2431
|
|
2478
2432
|
void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
|
2479
2433
|
PendingOp reason) {
|
2480
2434
|
if (grpc_call_trace.enabled()) {
|
2481
|
-
|
2482
|
-
|
2483
|
-
|
2484
|
-
std::vector<const char*> pending;
|
2485
|
-
for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) {
|
2486
|
-
if (static_cast<PendingOp>(i) == reason) continue;
|
2487
|
-
if (pending_op_bits & (1 << i)) {
|
2488
|
-
pending.push_back(PendingOpString(static_cast<PendingOp>(i)));
|
2489
|
-
}
|
2490
|
-
}
|
2491
|
-
gpr_log(
|
2492
|
-
GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s",
|
2493
|
-
DebugTag().c_str(), completion_info_[completion->index()].pending.tag,
|
2494
|
-
CompletionString(*completion).c_str(), PendingOpString(reason),
|
2495
|
-
(pending.empty()
|
2496
|
-
? (success ? std::string("done") : std::string("failed"))
|
2497
|
-
: absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ",")))
|
2498
|
-
.c_str());
|
2435
|
+
gpr_log(GPR_INFO, "%s[call] FinishOpOnCompletion completion:%s finish:%s",
|
2436
|
+
DebugTag().c_str(), CompletionString(*completion).c_str(),
|
2437
|
+
PendingOpString(reason));
|
2499
2438
|
}
|
2500
2439
|
const uint8_t i = completion->TakeIndex();
|
2501
2440
|
GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_));
|
2502
2441
|
CompletionInfo::Pending& pending = completion_info_[i].pending;
|
2503
|
-
|
2504
|
-
pending.
|
2505
|
-
|
2506
|
-
|
2507
|
-
|
2508
|
-
|
2509
|
-
|
2510
|
-
|
2511
|
-
|
2512
|
-
|
2513
|
-
|
2514
|
-
|
2442
|
+
bool success;
|
2443
|
+
switch (pending.RemovePendingBit(reason)) {
|
2444
|
+
case CompletionInfo::kPending:
|
2445
|
+
return; // Early out
|
2446
|
+
case CompletionInfo::kSuccess:
|
2447
|
+
success = true;
|
2448
|
+
break;
|
2449
|
+
case CompletionInfo::kFailure:
|
2450
|
+
success = false;
|
2451
|
+
break;
|
2452
|
+
}
|
2453
|
+
if (pending.is_recv_message && !success && *recv_message_ != nullptr) {
|
2454
|
+
grpc_byte_buffer_destroy(*recv_message_);
|
2455
|
+
*recv_message_ = nullptr;
|
2456
|
+
}
|
2457
|
+
auto error = success ? absl::OkStatus() : absl::CancelledError();
|
2458
|
+
if (pending.is_closure) {
|
2459
|
+
ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
|
2460
|
+
error);
|
2461
|
+
} else {
|
2462
|
+
grpc_cq_end_op(
|
2463
|
+
cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
|
2464
|
+
&completion_info_[i].completion);
|
2515
2465
|
}
|
2516
2466
|
}
|
2517
2467
|
|
2518
|
-
void PromiseBasedCall::Update() {
|
2519
|
-
keep_polling_ = false;
|
2520
|
-
do {
|
2521
|
-
UpdateOnce();
|
2522
|
-
} while (std::exchange(keep_polling_, false));
|
2523
|
-
}
|
2524
|
-
|
2525
|
-
void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; }
|
2526
|
-
|
2527
2468
|
void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) {
|
2528
|
-
MutexLock lock(&mu_);
|
2529
|
-
SetCompletionQueueLocked(cq);
|
2530
|
-
}
|
2531
|
-
|
2532
|
-
void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
|
2533
2469
|
cq_ = cq;
|
2534
2470
|
GRPC_CQ_INTERNAL_REF(cq, "bind");
|
2535
2471
|
call_context_.pollent_ =
|
@@ -2537,6 +2473,12 @@ void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
|
|
2537
2473
|
}
|
2538
2474
|
|
2539
2475
|
void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
|
2476
|
+
MutexLock lock(&deadline_mu_);
|
2477
|
+
if (grpc_call_trace.enabled()) {
|
2478
|
+
gpr_log(GPR_DEBUG, "%s[call] UpdateDeadline from=%s to=%s",
|
2479
|
+
DebugTag().c_str(), deadline_.ToString().c_str(),
|
2480
|
+
deadline.ToString().c_str());
|
2481
|
+
}
|
2540
2482
|
if (deadline >= deadline_) return;
|
2541
2483
|
auto* const event_engine = channel()->event_engine();
|
2542
2484
|
if (deadline_ != Timestamp::InfFuture()) {
|
@@ -2544,10 +2486,12 @@ void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
|
|
2544
2486
|
} else {
|
2545
2487
|
InternalRef("deadline");
|
2546
2488
|
}
|
2547
|
-
|
2489
|
+
deadline_ = deadline;
|
2490
|
+
deadline_task_ = event_engine->RunAfter(deadline - Timestamp::Now(), this);
|
2548
2491
|
}
|
2549
2492
|
|
2550
2493
|
void PromiseBasedCall::ResetDeadline() {
|
2494
|
+
MutexLock lock(&deadline_mu_);
|
2551
2495
|
if (deadline_ == Timestamp::InfFuture()) return;
|
2552
2496
|
auto* const event_engine = channel()->event_engine();
|
2553
2497
|
if (!event_engine->Cancel(deadline_task_)) return;
|
@@ -2564,117 +2508,88 @@ void PromiseBasedCall::Run() {
|
|
2564
2508
|
|
2565
2509
|
void PromiseBasedCall::StartSendMessage(const grpc_op& op,
|
2566
2510
|
const Completion& completion,
|
2567
|
-
PipeSender<MessageHandle>* sender
|
2568
|
-
|
2569
|
-
|
2570
|
-
|
2571
|
-
|
2572
|
-
|
2573
|
-
|
2574
|
-
|
2575
|
-
|
2576
|
-
|
2577
|
-
|
2578
|
-
|
2579
|
-
|
2580
|
-
|
2511
|
+
PipeSender<MessageHandle>* sender,
|
2512
|
+
Party::BulkSpawner& spawner) {
|
2513
|
+
QueueSend();
|
2514
|
+
SliceBuffer send;
|
2515
|
+
grpc_slice_buffer_swap(
|
2516
|
+
&op.data.send_message.send_message->data.raw.slice_buffer,
|
2517
|
+
send.c_slice_buffer());
|
2518
|
+
auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
|
2519
|
+
spawner.Spawn(
|
2520
|
+
"call_send_message",
|
2521
|
+
[this, sender, msg = std::move(msg)]() mutable {
|
2522
|
+
EnactSend();
|
2523
|
+
return sender->Push(std::move(msg));
|
2524
|
+
},
|
2525
|
+
[this, completion = AddOpToCompletion(
|
2526
|
+
completion, PendingOp::kSendMessage)](bool result) mutable {
|
2527
|
+
if (grpc_call_trace.enabled()) {
|
2528
|
+
gpr_log(GPR_DEBUG, "%sSendMessage completes %s", DebugTag().c_str(),
|
2529
|
+
result ? "successfully" : "with failure");
|
2530
|
+
}
|
2531
|
+
if (!result) FailCompletion(completion);
|
2532
|
+
FinishOpOnCompletion(&completion, PendingOp::kSendMessage);
|
2533
|
+
});
|
2581
2534
|
}
|
2582
2535
|
|
2583
|
-
|
2584
|
-
|
2585
|
-
|
2586
|
-
|
2587
|
-
|
2588
|
-
|
2589
|
-
|
2590
|
-
|
2591
|
-
if (!*result) {
|
2592
|
-
FailCompletion(send_message_completion_);
|
2593
|
-
return false;
|
2594
|
-
}
|
2595
|
-
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2596
|
-
outstanding_send_.reset();
|
2536
|
+
template <typename FirstPromiseFactory>
|
2537
|
+
void PromiseBasedCall::StartRecvMessage(
|
2538
|
+
const grpc_op& op, const Completion& completion,
|
2539
|
+
FirstPromiseFactory first_promise_factory,
|
2540
|
+
PipeReceiver<MessageHandle>* receiver, Party::BulkSpawner& spawner) {
|
2541
|
+
if (grpc_call_trace.enabled()) {
|
2542
|
+
gpr_log(GPR_INFO, "%s[call] Start RecvMessage: %s", DebugTag().c_str(),
|
2543
|
+
CompletionString(completion).c_str());
|
2597
2544
|
}
|
2598
|
-
return true;
|
2599
|
-
}
|
2600
|
-
|
2601
|
-
void PromiseBasedCall::CancelSendMessage() {
|
2602
|
-
if (!outstanding_send_.has_value()) return;
|
2603
|
-
FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
|
2604
|
-
outstanding_send_.reset();
|
2605
|
-
}
|
2606
|
-
|
2607
|
-
void PromiseBasedCall::StartRecvMessage(const grpc_op& op,
|
2608
|
-
const Completion& completion,
|
2609
|
-
PipeReceiver<MessageHandle>* receiver) {
|
2610
|
-
GPR_ASSERT(!outstanding_recv_.has_value());
|
2611
2545
|
recv_message_ = op.data.recv_message.recv_message;
|
2612
|
-
|
2613
|
-
|
2614
|
-
|
2615
|
-
|
2616
|
-
|
2617
|
-
|
2618
|
-
|
2619
|
-
|
2620
|
-
|
2621
|
-
|
2622
|
-
|
2623
|
-
|
2624
|
-
|
2625
|
-
|
2626
|
-
|
2627
|
-
|
2628
|
-
nullptr, 0
|
2629
|
-
|
2630
|
-
|
2631
|
-
|
2632
|
-
|
2633
|
-
|
2634
|
-
|
2635
|
-
|
2636
|
-
|
2637
|
-
|
2638
|
-
|
2639
|
-
|
2640
|
-
|
2641
|
-
|
2642
|
-
|
2643
|
-
|
2644
|
-
|
2645
|
-
|
2646
|
-
|
2647
|
-
|
2648
|
-
|
2649
|
-
|
2650
|
-
|
2651
|
-
|
2652
|
-
|
2653
|
-
|
2654
|
-
|
2655
|
-
|
2656
|
-
|
2657
|
-
|
2658
|
-
|
2659
|
-
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2660
|
-
} else if (completed_) {
|
2661
|
-
if (grpc_call_trace.enabled()) {
|
2662
|
-
gpr_log(GPR_INFO,
|
2663
|
-
"%s[call] UpdateOnce: outstanding_recv finishes: promise has "
|
2664
|
-
"completed without queuing a message, forcing end-of-stream",
|
2665
|
-
DebugTag().c_str());
|
2666
|
-
}
|
2667
|
-
outstanding_recv_.reset();
|
2668
|
-
*recv_message_ = nullptr;
|
2669
|
-
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2670
|
-
}
|
2671
|
-
}
|
2672
|
-
|
2673
|
-
void PromiseBasedCall::CancelRecvMessage() {
|
2674
|
-
if (!outstanding_recv_.has_value()) return;
|
2675
|
-
*recv_message_ = nullptr;
|
2676
|
-
outstanding_recv_.reset();
|
2677
|
-
FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
|
2546
|
+
spawner.Spawn(
|
2547
|
+
"call_recv_message",
|
2548
|
+
[first_promise_factory = std::move(first_promise_factory), receiver]() {
|
2549
|
+
return Seq(first_promise_factory(), receiver->Next());
|
2550
|
+
},
|
2551
|
+
[this,
|
2552
|
+
completion = AddOpToCompletion(completion, PendingOp::kReceiveMessage)](
|
2553
|
+
NextResult<MessageHandle> result) mutable {
|
2554
|
+
if (result.has_value()) {
|
2555
|
+
MessageHandle& message = *result;
|
2556
|
+
NoteLastMessageFlags(message->flags());
|
2557
|
+
if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
|
2558
|
+
(incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
|
2559
|
+
*recv_message_ = grpc_raw_compressed_byte_buffer_create(
|
2560
|
+
nullptr, 0, incoming_compression_algorithm());
|
2561
|
+
} else {
|
2562
|
+
*recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
|
2563
|
+
}
|
2564
|
+
grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
|
2565
|
+
&(*recv_message_)->data.raw.slice_buffer);
|
2566
|
+
if (grpc_call_trace.enabled()) {
|
2567
|
+
gpr_log(GPR_INFO,
|
2568
|
+
"%s[call] RecvMessage: outstanding_recv "
|
2569
|
+
"finishes: received %" PRIdPTR " byte message",
|
2570
|
+
DebugTag().c_str(),
|
2571
|
+
(*recv_message_)->data.raw.slice_buffer.length);
|
2572
|
+
}
|
2573
|
+
} else if (result.cancelled()) {
|
2574
|
+
if (grpc_call_trace.enabled()) {
|
2575
|
+
gpr_log(GPR_INFO,
|
2576
|
+
"%s[call] RecvMessage: outstanding_recv "
|
2577
|
+
"finishes: received end-of-stream with error",
|
2578
|
+
DebugTag().c_str());
|
2579
|
+
}
|
2580
|
+
FailCompletion(completion);
|
2581
|
+
*recv_message_ = nullptr;
|
2582
|
+
} else {
|
2583
|
+
if (grpc_call_trace.enabled()) {
|
2584
|
+
gpr_log(GPR_INFO,
|
2585
|
+
"%s[call] RecvMessage: outstanding_recv "
|
2586
|
+
"finishes: received end-of-stream",
|
2587
|
+
DebugTag().c_str());
|
2588
|
+
}
|
2589
|
+
*recv_message_ = nullptr;
|
2590
|
+
}
|
2591
|
+
FinishOpOnCompletion(&completion, PendingOp::kReceiveMessage);
|
2592
|
+
});
|
2678
2593
|
}
|
2679
2594
|
|
2680
2595
|
///////////////////////////////////////////////////////////////////////////////
|
@@ -2690,10 +2605,6 @@ void CallContext::IncrementRefCount(const char* reason) {
|
|
2690
2605
|
|
2691
2606
|
void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
|
2692
2607
|
|
2693
|
-
gpr_atm* CallContext::peer_string_atm_ptr() {
|
2694
|
-
return call_->peer_string_atm_ptr();
|
2695
|
-
}
|
2696
|
-
|
2697
2608
|
void CallContext::UpdateDeadline(Timestamp deadline) {
|
2698
2609
|
call_->UpdateDeadline(deadline);
|
2699
2610
|
}
|
@@ -2747,24 +2658,40 @@ class ClientPromiseBasedCall final : public PromiseBasedCall {
|
|
2747
2658
|
~ClientPromiseBasedCall() override {
|
2748
2659
|
ScopedContext context(this);
|
2749
2660
|
send_initial_metadata_.reset();
|
2750
|
-
|
2751
|
-
|
2752
|
-
//
|
2753
|
-
// out here and then allow the destructors to run at end of scope, but
|
2754
|
-
// before context.
|
2661
|
+
// Need to destroy the pipes under the ScopedContext above, so we
|
2662
|
+
// move them out here and then allow the destructors to run at
|
2663
|
+
// end of scope, but before context.
|
2755
2664
|
auto c2s = std::move(client_to_server_messages_);
|
2756
2665
|
auto s2c = std::move(server_to_client_messages_);
|
2757
2666
|
auto sim = std::move(server_initial_metadata_);
|
2758
2667
|
}
|
2759
2668
|
|
2760
|
-
absl::
|
2761
|
-
|
2762
|
-
|
2763
|
-
|
2764
|
-
|
2765
|
-
|
2669
|
+
void CancelWithError(absl::Status error) override {
|
2670
|
+
if (!started_.exchange(true, std::memory_order_relaxed)) {
|
2671
|
+
// Initial metadata not sent yet, so we can just fail the call.
|
2672
|
+
Spawn(
|
2673
|
+
"cancel_before_initial_metadata",
|
2674
|
+
[error = std::move(error), this]() {
|
2675
|
+
server_to_client_messages_.sender.Close();
|
2676
|
+
Finish(ServerMetadataFromStatus(error));
|
2677
|
+
return Empty{};
|
2678
|
+
},
|
2679
|
+
[](Empty) {});
|
2680
|
+
} else {
|
2681
|
+
Spawn(
|
2682
|
+
"cancel_with_error",
|
2683
|
+
[error = std::move(error), this]() {
|
2684
|
+
if (!cancel_error_.is_set()) {
|
2685
|
+
cancel_error_.Set(ServerMetadataFromStatus(error));
|
2686
|
+
}
|
2687
|
+
return Empty{};
|
2688
|
+
},
|
2689
|
+
[](Empty) {});
|
2690
|
+
}
|
2766
2691
|
}
|
2767
|
-
|
2692
|
+
absl::string_view GetServerAuthority() const override { abort(); }
|
2693
|
+
bool is_trailers_only() const override { return is_trailers_only_; }
|
2694
|
+
bool failed_before_recv_message() const override { return false; }
|
2768
2695
|
|
2769
2696
|
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
2770
2697
|
bool is_notify_tag_closure) override;
|
@@ -2774,65 +2701,79 @@ class ClientPromiseBasedCall final : public PromiseBasedCall {
|
|
2774
2701
|
}
|
2775
2702
|
|
2776
2703
|
private:
|
2777
|
-
// Poll the underlying promise (and sundry objects) once.
|
2778
|
-
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
2779
2704
|
// Finish the call with the given status/trailing metadata.
|
2780
|
-
void Finish(ServerMetadataHandle trailing_metadata)
|
2781
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2705
|
+
void Finish(ServerMetadataHandle trailing_metadata);
|
2782
2706
|
// Validate that a set of ops is valid for a client call.
|
2783
|
-
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const
|
2784
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2707
|
+
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
2785
2708
|
// Commit a valid batch of operations to be executed.
|
2786
2709
|
void CommitBatch(const grpc_op* ops, size_t nops,
|
2787
|
-
const Completion& completion)
|
2788
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2710
|
+
const Completion& completion);
|
2789
2711
|
// Start the underlying promise.
|
2790
|
-
void StartPromise(ClientMetadataHandle client_initial_metadata
|
2791
|
-
|
2712
|
+
void StartPromise(ClientMetadataHandle client_initial_metadata,
|
2713
|
+
const Completion& completion, Party::BulkSpawner& spawner);
|
2714
|
+
// Start receiving initial metadata
|
2715
|
+
void StartRecvInitialMetadata(grpc_metadata_array* array,
|
2716
|
+
const Completion& completion,
|
2717
|
+
Party::BulkSpawner& spawner);
|
2718
|
+
void StartRecvStatusOnClient(
|
2719
|
+
const Completion& completion,
|
2720
|
+
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2721
|
+
Party::BulkSpawner& spawner);
|
2792
2722
|
// Publish status out to the application.
|
2793
2723
|
void PublishStatus(
|
2794
2724
|
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
2795
|
-
ServerMetadataHandle trailing_metadata)
|
2796
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2725
|
+
ServerMetadataHandle trailing_metadata);
|
2797
2726
|
// Publish server initial metadata out to the application.
|
2798
|
-
void PublishInitialMetadata(ServerMetadata* metadata)
|
2799
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
2800
|
-
|
2801
|
-
ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
|
2802
|
-
Pipe<ServerMetadataHandle> server_initial_metadata_ ABSL_GUARDED_BY(mu()){
|
2803
|
-
arena()};
|
2804
|
-
Pipe<MessageHandle> client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2805
|
-
Pipe<MessageHandle> server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()};
|
2727
|
+
void PublishInitialMetadata(ServerMetadata* metadata);
|
2806
2728
|
|
2807
2729
|
ClientMetadataHandle send_initial_metadata_;
|
2808
|
-
|
2809
|
-
|
2810
|
-
|
2811
|
-
|
2812
|
-
|
2813
|
-
|
2814
|
-
|
2815
|
-
|
2816
|
-
|
2817
|
-
|
2818
|
-
|
2819
|
-
bool
|
2730
|
+
Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
|
2731
|
+
Latch<ServerMetadataHandle> server_trailing_metadata_;
|
2732
|
+
Latch<ServerMetadataHandle> cancel_error_;
|
2733
|
+
Pipe<MessageHandle> client_to_server_messages_{arena()};
|
2734
|
+
Pipe<MessageHandle> server_to_client_messages_{arena()};
|
2735
|
+
bool is_trailers_only_;
|
2736
|
+
// True once the promise for the call is started.
|
2737
|
+
// This corresponds to sending initial metadata, or cancelling before doing
|
2738
|
+
// so.
|
2739
|
+
// In the latter case real world code sometimes does not sent the initial
|
2740
|
+
// metadata, and so gating based upon that does not work out.
|
2741
|
+
std::atomic<bool> started_{false};
|
2820
2742
|
};
|
2821
2743
|
|
2822
2744
|
void ClientPromiseBasedCall::StartPromise(
|
2823
|
-
ClientMetadataHandle client_initial_metadata
|
2824
|
-
|
2825
|
-
|
2826
|
-
|
2827
|
-
|
2828
|
-
|
2829
|
-
|
2830
|
-
|
2831
|
-
|
2832
|
-
|
2833
|
-
|
2834
|
-
|
2835
|
-
|
2745
|
+
ClientMetadataHandle client_initial_metadata, const Completion& completion,
|
2746
|
+
Party::BulkSpawner& spawner) {
|
2747
|
+
auto token = ClientInitialMetadataOutstandingToken::New(arena());
|
2748
|
+
spawner.Spawn(
|
2749
|
+
"call_send_initial_metadata", token.Wait(),
|
2750
|
+
[this,
|
2751
|
+
completion = AddOpToCompletion(
|
2752
|
+
completion, PendingOp::kSendInitialMetadata)](bool result) mutable {
|
2753
|
+
if (!result) FailCompletion(completion);
|
2754
|
+
FinishOpOnCompletion(&completion, PendingOp::kSendInitialMetadata);
|
2755
|
+
});
|
2756
|
+
spawner.Spawn(
|
2757
|
+
"client_promise",
|
2758
|
+
[this, client_initial_metadata = std::move(client_initial_metadata),
|
2759
|
+
token = std::move(token)]() mutable {
|
2760
|
+
return Race(
|
2761
|
+
cancel_error_.Wait(),
|
2762
|
+
Map(channel()->channel_stack()->MakeClientCallPromise(
|
2763
|
+
CallArgs{std::move(client_initial_metadata),
|
2764
|
+
std::move(token), &server_initial_metadata_.sender,
|
2765
|
+
&client_to_server_messages_.receiver,
|
2766
|
+
&server_to_client_messages_.sender}),
|
2767
|
+
[this](ServerMetadataHandle trailing_metadata) {
|
2768
|
+
// If we're cancelled the transport doesn't get to return
|
2769
|
+
// stats.
|
2770
|
+
AcceptTransportStatsFromContext();
|
2771
|
+
return trailing_metadata;
|
2772
|
+
}));
|
2773
|
+
},
|
2774
|
+
[this](ServerMetadataHandle trailing_metadata) {
|
2775
|
+
Finish(std::move(trailing_metadata));
|
2776
|
+
});
|
2836
2777
|
}
|
2837
2778
|
|
2838
2779
|
grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
@@ -2873,49 +2814,61 @@ grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
|
2873
2814
|
|
2874
2815
|
void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
2875
2816
|
const Completion& completion) {
|
2817
|
+
Party::BulkSpawner spawner(this);
|
2876
2818
|
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
2877
2819
|
const grpc_op& op = ops[op_idx];
|
2878
2820
|
switch (op.op) {
|
2879
2821
|
case GRPC_OP_SEND_INITIAL_METADATA: {
|
2880
|
-
|
2881
|
-
|
2882
|
-
|
2883
|
-
|
2884
|
-
|
2885
|
-
|
2886
|
-
|
2887
|
-
StartPromise(std::move(send_initial_metadata_));
|
2822
|
+
if (started_.exchange(true, std::memory_order_relaxed)) break;
|
2823
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
2824
|
+
op.data.send_initial_metadata.count,
|
2825
|
+
send_initial_metadata_.get());
|
2826
|
+
PrepareOutgoingInitialMetadata(op, *send_initial_metadata_);
|
2827
|
+
if (send_deadline() != Timestamp::InfFuture()) {
|
2828
|
+
send_initial_metadata_->Set(GrpcTimeoutMetadata(), send_deadline());
|
2888
2829
|
}
|
2830
|
+
send_initial_metadata_->Set(
|
2831
|
+
WaitForReady(),
|
2832
|
+
WaitForReady::ValueType{
|
2833
|
+
(op.flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
|
2834
|
+
(op.flags &
|
2835
|
+
GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
|
2836
|
+
StartPromise(std::move(send_initial_metadata_), completion, spawner);
|
2889
2837
|
} break;
|
2890
2838
|
case GRPC_OP_RECV_INITIAL_METADATA: {
|
2891
|
-
|
2892
|
-
op.data.recv_initial_metadata.recv_initial_metadata
|
2893
|
-
|
2894
|
-
server_initial_metadata_.receiver.Next());
|
2895
|
-
recv_initial_metadata_completion_ =
|
2896
|
-
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata);
|
2839
|
+
StartRecvInitialMetadata(
|
2840
|
+
op.data.recv_initial_metadata.recv_initial_metadata, completion,
|
2841
|
+
spawner);
|
2897
2842
|
} break;
|
2898
2843
|
case GRPC_OP_RECV_STATUS_ON_CLIENT: {
|
2899
|
-
|
2900
|
-
|
2901
|
-
if (auto* finished_metadata =
|
2902
|
-
absl::get_if<ServerMetadataHandle>(&recv_status_on_client_)) {
|
2903
|
-
PublishStatus(op.data.recv_status_on_client,
|
2904
|
-
std::move(*finished_metadata));
|
2905
|
-
} else {
|
2906
|
-
recv_status_on_client_ = op.data.recv_status_on_client;
|
2907
|
-
}
|
2844
|
+
StartRecvStatusOnClient(completion, op.data.recv_status_on_client,
|
2845
|
+
spawner);
|
2908
2846
|
} break;
|
2909
2847
|
case GRPC_OP_SEND_MESSAGE:
|
2910
|
-
StartSendMessage(op, completion, &client_to_server_messages_.sender
|
2848
|
+
StartSendMessage(op, completion, &client_to_server_messages_.sender,
|
2849
|
+
spawner);
|
2911
2850
|
break;
|
2912
2851
|
case GRPC_OP_RECV_MESSAGE:
|
2913
|
-
StartRecvMessage(
|
2852
|
+
StartRecvMessage(
|
2853
|
+
op, completion,
|
2854
|
+
[this]() {
|
2855
|
+
return server_initial_metadata_.receiver.AwaitClosed();
|
2856
|
+
},
|
2857
|
+
&server_to_client_messages_.receiver, spawner);
|
2914
2858
|
break;
|
2915
2859
|
case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
|
2916
|
-
|
2917
|
-
|
2918
|
-
|
2860
|
+
spawner.Spawn(
|
2861
|
+
"send_close_from_client",
|
2862
|
+
[this]() {
|
2863
|
+
client_to_server_messages_.sender.Close();
|
2864
|
+
return Empty{};
|
2865
|
+
},
|
2866
|
+
[this,
|
2867
|
+
completion = AddOpToCompletion(
|
2868
|
+
completion, PendingOp::kSendCloseFromClient)](Empty) mutable {
|
2869
|
+
FinishOpOnCompletion(&completion,
|
2870
|
+
PendingOp::kSendCloseFromClient);
|
2871
|
+
});
|
2919
2872
|
break;
|
2920
2873
|
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
2921
2874
|
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
@@ -2928,8 +2881,6 @@ grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
|
2928
2881
|
size_t nops,
|
2929
2882
|
void* notify_tag,
|
2930
2883
|
bool is_notify_tag_closure) {
|
2931
|
-
MutexLock lock(mu());
|
2932
|
-
ScopedContext activity_context(this);
|
2933
2884
|
if (nops == 0) {
|
2934
2885
|
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
2935
2886
|
return GRPC_CALL_OK;
|
@@ -2941,69 +2892,35 @@ grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
|
|
2941
2892
|
Completion completion =
|
2942
2893
|
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
2943
2894
|
CommitBatch(ops, nops, completion);
|
2944
|
-
Update();
|
2945
2895
|
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
2946
2896
|
return GRPC_CALL_OK;
|
2947
2897
|
}
|
2948
2898
|
|
2949
|
-
void ClientPromiseBasedCall::
|
2950
|
-
|
2951
|
-
|
2952
|
-
|
2953
|
-
|
2954
|
-
|
2955
|
-
|
2956
|
-
|
2957
|
-
|
2958
|
-
|
2959
|
-
|
2960
|
-
|
2961
|
-
|
2962
|
-
|
2963
|
-
|
2964
|
-
|
2965
|
-
|
2966
|
-
|
2967
|
-
|
2968
|
-
|
2969
|
-
|
2970
|
-
|
2971
|
-
|
2972
|
-
|
2973
|
-
|
2974
|
-
if (auto* server_initial_metadata = r.value_if_ready()) {
|
2975
|
-
PublishInitialMetadata(server_initial_metadata->value().get());
|
2976
|
-
} else if (completed()) {
|
2977
|
-
ServerMetadata no_metadata{GetContext<Arena>()};
|
2978
|
-
PublishInitialMetadata(&no_metadata);
|
2979
|
-
}
|
2980
|
-
}
|
2981
|
-
if (!PollSendMessage()) {
|
2982
|
-
Finish(ServerMetadataFromStatus(absl::Status(
|
2983
|
-
absl::StatusCode::kInternal, "Failed to send message to server")));
|
2984
|
-
}
|
2985
|
-
if (!is_sending() && close_send_completion_.has_value()) {
|
2986
|
-
client_to_server_messages_.sender.Close();
|
2987
|
-
FinishOpOnCompletion(&close_send_completion_,
|
2988
|
-
PendingOp::kSendCloseFromClient);
|
2989
|
-
}
|
2990
|
-
if (promise_.has_value()) {
|
2991
|
-
Poll<ServerMetadataHandle> r = promise_();
|
2992
|
-
if (grpc_call_trace.enabled()) {
|
2993
|
-
gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
|
2994
|
-
DebugTag().c_str(),
|
2995
|
-
PollToString(r, [](const ServerMetadataHandle& h) {
|
2996
|
-
return h->DebugString();
|
2997
|
-
}).c_str());
|
2998
|
-
}
|
2999
|
-
if (auto* result = r.value_if_ready()) {
|
3000
|
-
AcceptTransportStatsFromContext();
|
3001
|
-
Finish(std::move(*result));
|
3002
|
-
}
|
3003
|
-
}
|
3004
|
-
if (incoming_compression_algorithm_.has_value()) {
|
3005
|
-
PollRecvMessage(*incoming_compression_algorithm_);
|
3006
|
-
}
|
2899
|
+
void ClientPromiseBasedCall::StartRecvInitialMetadata(
|
2900
|
+
grpc_metadata_array* array, const Completion& completion,
|
2901
|
+
Party::BulkSpawner& spawner) {
|
2902
|
+
spawner.Spawn(
|
2903
|
+
"recv_initial_metadata",
|
2904
|
+
Race(server_initial_metadata_.receiver.Next(),
|
2905
|
+
Map(finished(),
|
2906
|
+
[](Empty) { return NextResult<ServerMetadataHandle>(true); })),
|
2907
|
+
[this, array,
|
2908
|
+
completion =
|
2909
|
+
AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata)](
|
2910
|
+
NextResult<ServerMetadataHandle> next_metadata) mutable {
|
2911
|
+
server_initial_metadata_.sender.Close();
|
2912
|
+
ServerMetadataHandle metadata;
|
2913
|
+
if (next_metadata.has_value()) {
|
2914
|
+
is_trailers_only_ = false;
|
2915
|
+
metadata = std::move(next_metadata.value());
|
2916
|
+
} else {
|
2917
|
+
is_trailers_only_ = true;
|
2918
|
+
metadata = arena()->MakePooled<ServerMetadata>(arena());
|
2919
|
+
}
|
2920
|
+
ProcessIncomingInitialMetadata(*metadata);
|
2921
|
+
PublishMetadataArray(metadata.get(), array);
|
2922
|
+
FinishOpOnCompletion(&completion, PendingOp::kReceiveInitialMetadata);
|
2923
|
+
});
|
3007
2924
|
}
|
3008
2925
|
|
3009
2926
|
void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
@@ -3011,31 +2928,9 @@ void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
|
3011
2928
|
gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
|
3012
2929
|
trailing_metadata->DebugString().c_str());
|
3013
2930
|
}
|
3014
|
-
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3015
2931
|
ResetDeadline();
|
3016
2932
|
set_completed();
|
3017
|
-
|
3018
|
-
ForceImmediateRepoll();
|
3019
|
-
}
|
3020
|
-
const bool pending_initial_metadata =
|
3021
|
-
server_initial_metadata_ready_.has_value();
|
3022
|
-
if (!pending_initial_metadata) {
|
3023
|
-
server_initial_metadata_ready_.emplace(
|
3024
|
-
server_initial_metadata_.receiver.Next());
|
3025
|
-
}
|
3026
|
-
Poll<NextResult<ServerMetadataHandle>> r =
|
3027
|
-
(*server_initial_metadata_ready_)();
|
3028
|
-
server_initial_metadata_ready_.reset();
|
3029
|
-
if (auto* result = r.value_if_ready()) {
|
3030
|
-
if (pending_initial_metadata) PublishInitialMetadata(result->value().get());
|
3031
|
-
is_trailers_only_ = false;
|
3032
|
-
} else {
|
3033
|
-
if (pending_initial_metadata) {
|
3034
|
-
ServerMetadata no_metadata{GetContext<Arena>()};
|
3035
|
-
PublishInitialMetadata(&no_metadata);
|
3036
|
-
}
|
3037
|
-
is_trailers_only_ = true;
|
3038
|
-
}
|
2933
|
+
client_to_server_messages_.sender.Close();
|
3039
2934
|
if (auto* channelz_channel = channel()->channelz_node()) {
|
3040
2935
|
if (trailing_metadata->get(GrpcStatusMetadata())
|
3041
2936
|
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
@@ -3044,13 +2939,7 @@ void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
|
|
3044
2939
|
channelz_channel->RecordCallFailed();
|
3045
2940
|
}
|
3046
2941
|
}
|
3047
|
-
|
3048
|
-
absl::get_if<grpc_op::grpc_op_data::grpc_op_recv_status_on_client>(
|
3049
|
-
&recv_status_on_client_)) {
|
3050
|
-
PublishStatus(*status_request, std::move(trailing_metadata));
|
3051
|
-
} else {
|
3052
|
-
recv_status_on_client_ = std::move(trailing_metadata);
|
3053
|
-
}
|
2942
|
+
server_trailing_metadata_.Set(std::move(trailing_metadata));
|
3054
2943
|
}
|
3055
2944
|
|
3056
2945
|
namespace {
|
@@ -3076,35 +2965,43 @@ std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
|
|
3076
2965
|
}
|
3077
2966
|
} // namespace
|
3078
2967
|
|
3079
|
-
void ClientPromiseBasedCall::
|
2968
|
+
void ClientPromiseBasedCall::StartRecvStatusOnClient(
|
2969
|
+
const Completion& completion,
|
3080
2970
|
grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
|
3081
|
-
|
3082
|
-
|
3083
|
-
|
3084
|
-
|
3085
|
-
|
3086
|
-
|
3087
|
-
|
3088
|
-
|
3089
|
-
|
3090
|
-
|
3091
|
-
|
3092
|
-
|
3093
|
-
|
3094
|
-
|
3095
|
-
|
3096
|
-
|
3097
|
-
|
3098
|
-
|
3099
|
-
|
3100
|
-
|
3101
|
-
|
3102
|
-
|
3103
|
-
|
3104
|
-
|
3105
|
-
|
3106
|
-
|
3107
|
-
|
2971
|
+
Party::BulkSpawner& spawner) {
|
2972
|
+
ForceCompletionSuccess(completion);
|
2973
|
+
spawner.Spawn(
|
2974
|
+
"recv_status_on_client", server_trailing_metadata_.Wait(),
|
2975
|
+
[this, op_args,
|
2976
|
+
completion =
|
2977
|
+
AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient)](
|
2978
|
+
ServerMetadataHandle trailing_metadata) mutable {
|
2979
|
+
const grpc_status_code status =
|
2980
|
+
trailing_metadata->get(GrpcStatusMetadata())
|
2981
|
+
.value_or(GRPC_STATUS_UNKNOWN);
|
2982
|
+
*op_args.status = status;
|
2983
|
+
absl::string_view message_string;
|
2984
|
+
if (Slice* message =
|
2985
|
+
trailing_metadata->get_pointer(GrpcMessageMetadata())) {
|
2986
|
+
message_string = message->as_string_view();
|
2987
|
+
*op_args.status_details = message->Ref().TakeCSlice();
|
2988
|
+
} else {
|
2989
|
+
*op_args.status_details = grpc_empty_slice();
|
2990
|
+
}
|
2991
|
+
if (message_string.empty()) {
|
2992
|
+
RunFinalization(status, nullptr);
|
2993
|
+
} else {
|
2994
|
+
std::string error_string(message_string);
|
2995
|
+
RunFinalization(status, error_string.c_str());
|
2996
|
+
}
|
2997
|
+
if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
|
2998
|
+
*op_args.error_string =
|
2999
|
+
gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
|
3000
|
+
}
|
3001
|
+
PublishMetadataArray(trailing_metadata.get(),
|
3002
|
+
op_args.trailing_metadata);
|
3003
|
+
FinishOpOnCompletion(&completion, PendingOp::kReceiveStatusOnClient);
|
3004
|
+
});
|
3108
3005
|
}
|
3109
3006
|
#endif
|
3110
3007
|
|
@@ -3117,19 +3014,18 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
|
|
3117
3014
|
public:
|
3118
3015
|
ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
|
3119
3016
|
|
3120
|
-
void
|
3121
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3017
|
+
void CancelWithError(grpc_error_handle) override;
|
3122
3018
|
grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
|
3123
3019
|
bool is_notify_tag_closure) override;
|
3124
|
-
bool failed_before_recv_message() const override {
|
3020
|
+
bool failed_before_recv_message() const override { return false; }
|
3125
3021
|
bool is_trailers_only() const override { abort(); }
|
3126
3022
|
absl::string_view GetServerAuthority() const override { return ""; }
|
3127
3023
|
|
3128
3024
|
// Polling order for the server promise stack:
|
3129
3025
|
//
|
3130
3026
|
// │ ┌───────────────────────────────────────┐
|
3131
|
-
// │ │ ServerPromiseBasedCall
|
3132
|
-
// │ ├───────────────────────────────────────┤
|
3027
|
+
// │ │ ServerPromiseBasedCall ├──► Lifetime management
|
3028
|
+
// │ ├───────────────────────────────────────┤
|
3133
3029
|
// │ │ ConnectedChannel ├─┐
|
3134
3030
|
// │ ├───────────────────────────────────────┤ └► Interactions with the
|
3135
3031
|
// │ │ ... closest to transport filter │ transport - send/recv msgs
|
@@ -3140,16 +3036,12 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
|
|
3140
3036
|
// │ ├───────────────────────────────────────┤ │ setup, publishing call to
|
3141
3037
|
// │ │ Server::ChannelData::MakeCallPromise ├─┘ application
|
3142
3038
|
// │ ├───────────────────────────────────────┤
|
3143
|
-
// │ │
|
3144
|
-
// ▼ └───────────────────────────────────────┘
|
3145
|
-
// Polling &
|
3039
|
+
// │ │ MakeTopOfServerCallPromise ├──► Send trailing metadata
|
3040
|
+
// ▼ └───────────────────────────────────────┘
|
3041
|
+
// Polling &
|
3146
3042
|
// instantiation
|
3147
3043
|
// order
|
3148
3044
|
|
3149
|
-
void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
|
3150
|
-
Poll<ServerMetadataHandle> PollTopOfCall()
|
3151
|
-
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
|
3152
|
-
|
3153
3045
|
std::string DebugTag() const override {
|
3154
3046
|
return absl::StrFormat("SERVER_CALL[%p]: ", this);
|
3155
3047
|
}
|
@@ -3159,44 +3051,64 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
|
|
3159
3051
|
private:
|
3160
3052
|
class RecvCloseOpCancelState {
|
3161
3053
|
public:
|
3162
|
-
// Request that receiver be filled in per
|
3163
|
-
// Returns true if the request can
|
3164
|
-
// Returns false if the request will be
|
3054
|
+
// Request that receiver be filled in per
|
3055
|
+
// grpc_op_recv_close_on_server. Returns true if the request can
|
3056
|
+
// be fulfilled immediately. Returns false if the request will be
|
3057
|
+
// fulfilled later.
|
3165
3058
|
bool ReceiveCloseOnServerOpStarted(int* receiver) {
|
3166
|
-
|
3167
|
-
|
3168
|
-
|
3169
|
-
|
3170
|
-
|
3171
|
-
|
3172
|
-
|
3173
|
-
|
3174
|
-
|
3175
|
-
|
3176
|
-
|
3177
|
-
|
3178
|
-
|
3059
|
+
uintptr_t state = state_.load(std::memory_order_acquire);
|
3060
|
+
uintptr_t new_state;
|
3061
|
+
do {
|
3062
|
+
switch (state) {
|
3063
|
+
case kUnset:
|
3064
|
+
new_state = reinterpret_cast<uintptr_t>(receiver);
|
3065
|
+
break;
|
3066
|
+
case kFinishedWithFailure:
|
3067
|
+
*receiver = 1;
|
3068
|
+
return true;
|
3069
|
+
case kFinishedWithSuccess:
|
3070
|
+
*receiver = 0;
|
3071
|
+
return true;
|
3072
|
+
default:
|
3073
|
+
Crash("Two threads offered ReceiveCloseOnServerOpStarted");
|
3074
|
+
}
|
3075
|
+
} while (!state_.compare_exchange_weak(state, new_state,
|
3076
|
+
std::memory_order_acq_rel,
|
3077
|
+
std::memory_order_acquire));
|
3078
|
+
return false;
|
3179
3079
|
}
|
3180
3080
|
|
3181
3081
|
// Mark the call as having completed.
|
3182
|
-
// Returns true if this finishes a previous
|
3183
|
-
|
3184
|
-
|
3185
|
-
|
3186
|
-
|
3187
|
-
|
3188
|
-
|
3189
|
-
|
3190
|
-
|
3191
|
-
|
3192
|
-
|
3193
|
-
|
3194
|
-
|
3195
|
-
|
3082
|
+
// Returns true if this finishes a previous
|
3083
|
+
// RequestReceiveCloseOnServer.
|
3084
|
+
bool CompleteCallWithCancelledSetTo(bool cancelled) {
|
3085
|
+
uintptr_t state = state_.load(std::memory_order_acquire);
|
3086
|
+
uintptr_t new_state;
|
3087
|
+
bool r;
|
3088
|
+
do {
|
3089
|
+
switch (state) {
|
3090
|
+
case kUnset:
|
3091
|
+
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3092
|
+
r = false;
|
3093
|
+
break;
|
3094
|
+
case kFinishedWithFailure:
|
3095
|
+
return false;
|
3096
|
+
case kFinishedWithSuccess:
|
3097
|
+
Crash("unreachable");
|
3098
|
+
default:
|
3099
|
+
new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
|
3100
|
+
r = true;
|
3101
|
+
}
|
3102
|
+
} while (!state_.compare_exchange_weak(state, new_state,
|
3103
|
+
std::memory_order_acq_rel,
|
3104
|
+
std::memory_order_acquire));
|
3105
|
+
if (r) *reinterpret_cast<int*>(state) = cancelled ? 1 : 0;
|
3106
|
+
return r;
|
3196
3107
|
}
|
3197
3108
|
|
3198
3109
|
std::string ToString() const {
|
3199
|
-
|
3110
|
+
auto state = state_.load(std::memory_order_relaxed);
|
3111
|
+
switch (state) {
|
3200
3112
|
case kUnset:
|
3201
3113
|
return "Unset";
|
3202
3114
|
case kFinishedWithFailure:
|
@@ -3205,7 +3117,7 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
|
|
3205
3117
|
return "FinishedWithSuccess";
|
3206
3118
|
default:
|
3207
3119
|
return absl::StrFormat("WaitingForReceiver(%p)",
|
3208
|
-
reinterpret_cast<void*>(
|
3120
|
+
reinterpret_cast<void*>(state));
|
3209
3121
|
}
|
3210
3122
|
}
|
3211
3123
|
|
@@ -3213,37 +3125,28 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
|
|
3213
3125
|
static constexpr uintptr_t kUnset = 0;
|
3214
3126
|
static constexpr uintptr_t kFinishedWithFailure = 1;
|
3215
3127
|
static constexpr uintptr_t kFinishedWithSuccess = 2;
|
3216
|
-
// Holds one of kUnset, kFinishedWithFailure, or
|
3217
|
-
// OR an int* that wants to receive the
|
3218
|
-
|
3128
|
+
// Holds one of kUnset, kFinishedWithFailure, or
|
3129
|
+
// kFinishedWithSuccess OR an int* that wants to receive the
|
3130
|
+
// final status.
|
3131
|
+
std::atomic<uintptr_t> state_{kUnset};
|
3219
3132
|
};
|
3220
3133
|
|
3221
3134
|
grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
|
3222
3135
|
void CommitBatch(const grpc_op* ops, size_t nops,
|
3223
|
-
const Completion& completion)
|
3224
|
-
|
3136
|
+
const Completion& completion);
|
3137
|
+
void Finish(ServerMetadataHandle result);
|
3225
3138
|
|
3226
3139
|
friend class ServerCallContext;
|
3227
3140
|
ServerCallContext call_context_;
|
3228
3141
|
Server* const server_;
|
3229
|
-
|
3230
|
-
PipeSender<MessageHandle>* server_to_client_messages_
|
3231
|
-
|
3232
|
-
|
3233
|
-
|
3234
|
-
|
3235
|
-
|
3236
|
-
|
3237
|
-
SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) =
|
3238
|
-
absl::monostate{};
|
3239
|
-
ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu());
|
3240
|
-
grpc_compression_algorithm incoming_compression_algorithm_
|
3241
|
-
ABSL_GUARDED_BY(mu());
|
3242
|
-
RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu());
|
3243
|
-
Completion recv_close_completion_ ABSL_GUARDED_BY(mu());
|
3244
|
-
bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false;
|
3245
|
-
Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu());
|
3246
|
-
ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu());
|
3142
|
+
PipeSender<ServerMetadataHandle>* server_initial_metadata_ = nullptr;
|
3143
|
+
PipeSender<MessageHandle>* server_to_client_messages_ = nullptr;
|
3144
|
+
PipeReceiver<MessageHandle>* client_to_server_messages_ = nullptr;
|
3145
|
+
Latch<ServerMetadataHandle> send_trailing_metadata_;
|
3146
|
+
RecvCloseOpCancelState recv_close_op_cancel_state_;
|
3147
|
+
ClientMetadataHandle client_initial_metadata_;
|
3148
|
+
Completion recv_close_completion_;
|
3149
|
+
std::atomic<bool> cancelled_{false};
|
3247
3150
|
};
|
3248
3151
|
|
3249
3152
|
ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
@@ -3256,106 +3159,60 @@ ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
|
|
3256
3159
|
if (channelz_node != nullptr) {
|
3257
3160
|
channelz_node->RecordCallStarted();
|
3258
3161
|
}
|
3259
|
-
|
3162
|
+
// TODO(yashykt): In the future, we want to also enable stats and trace
|
3163
|
+
// collecting from when the call is created at the transport. The idea is that
|
3164
|
+
// the transport would create the call tracer and pass it in as part of the
|
3165
|
+
// metadata.
|
3166
|
+
auto* server_call_tracer_factory =
|
3167
|
+
ServerCallTracerFactory::Get(args->server->channel_args());
|
3168
|
+
if (server_call_tracer_factory != nullptr) {
|
3169
|
+
auto* server_call_tracer =
|
3170
|
+
server_call_tracer_factory->CreateNewServerCallTracer(arena);
|
3171
|
+
if (server_call_tracer != nullptr) {
|
3172
|
+
// Note that we are setting both
|
3173
|
+
// GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
|
3174
|
+
// GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
|
3175
|
+
// promise-based world, we would just a single tracer object for each
|
3176
|
+
// stack (call, subchannel_call, server_call.)
|
3177
|
+
ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
|
3178
|
+
server_call_tracer, nullptr);
|
3179
|
+
ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
|
3180
|
+
}
|
3181
|
+
}
|
3260
3182
|
ScopedContext activity_context(this);
|
3261
|
-
|
3262
|
-
|
3183
|
+
Spawn("server_promise",
|
3184
|
+
channel()->channel_stack()->MakeServerCallPromise(
|
3185
|
+
CallArgs{nullptr, ClientInitialMetadataOutstandingToken::Empty(),
|
3186
|
+
nullptr, nullptr, nullptr}),
|
3187
|
+
[this](ServerMetadataHandle result) { Finish(std::move(result)); });
|
3263
3188
|
}
|
3264
3189
|
|
3265
|
-
|
3190
|
+
void ServerPromiseBasedCall::Finish(ServerMetadataHandle result) {
|
3266
3191
|
if (grpc_call_trace.enabled()) {
|
3267
|
-
gpr_log(GPR_INFO, "%s[call]
|
3268
|
-
|
3269
|
-
|
3270
|
-
? absl::StrCat("send-metadata:",
|
3271
|
-
send_trailing_metadata_->DebugString(), " ")
|
3272
|
-
.c_str()
|
3273
|
-
: " ",
|
3274
|
-
PollStateDebugString().c_str());
|
3192
|
+
gpr_log(GPR_INFO, "%s[call] Finish: recv_close_state:%s result:%s",
|
3193
|
+
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3194
|
+
result->DebugString().c_str());
|
3275
3195
|
}
|
3276
|
-
|
3277
|
-
|
3278
|
-
|
3279
|
-
|
3280
|
-
}
|
3281
|
-
|
3282
|
-
PollSendMessage();
|
3283
|
-
PollRecvMessage(incoming_compression_algorithm_);
|
3284
|
-
|
3285
|
-
if (!is_sending() && send_trailing_metadata_ != nullptr) {
|
3286
|
-
server_to_client_messages_->Close();
|
3287
|
-
return std::move(send_trailing_metadata_);
|
3196
|
+
if (recv_close_op_cancel_state_.CompleteCallWithCancelledSetTo(
|
3197
|
+
result->get(GrpcCallWasCancelled()).value_or(true))) {
|
3198
|
+
FinishOpOnCompletion(&recv_close_completion_,
|
3199
|
+
PendingOp::kReceiveCloseOnServer);
|
3288
3200
|
}
|
3289
|
-
|
3290
|
-
|
3291
|
-
}
|
3292
|
-
|
3293
|
-
void ServerPromiseBasedCall::UpdateOnce() {
|
3294
|
-
if (grpc_call_trace.enabled()) {
|
3295
|
-
gpr_log(
|
3296
|
-
GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s",
|
3297
|
-
DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
|
3298
|
-
recv_close_completion_.has_value()
|
3299
|
-
? absl::StrCat(":", CompletionString(recv_close_completion_))
|
3300
|
-
.c_str()
|
3301
|
-
: "",
|
3302
|
-
send_status_from_server_completion_.has_value()
|
3303
|
-
? absl::StrCat(
|
3304
|
-
"send_status:",
|
3305
|
-
CompletionString(send_status_from_server_completion_), " ")
|
3306
|
-
.c_str()
|
3307
|
-
: "",
|
3308
|
-
PollStateDebugString().c_str(),
|
3309
|
-
promise_.has_value() ? "true" : "false");
|
3310
|
-
}
|
3311
|
-
if (auto* p =
|
3312
|
-
absl::get_if<typename PipeSender<ServerMetadataHandle>::PushType>(
|
3313
|
-
&send_initial_metadata_state_)) {
|
3314
|
-
if ((*p)().ready()) {
|
3315
|
-
send_initial_metadata_state_ = absl::monostate{};
|
3316
|
-
}
|
3201
|
+
if (server_initial_metadata_ != nullptr) {
|
3202
|
+
server_initial_metadata_->Close();
|
3317
3203
|
}
|
3318
|
-
|
3319
|
-
|
3320
|
-
if (
|
3321
|
-
|
3322
|
-
|
3323
|
-
|
3324
|
-
|
3325
|
-
}).c_str());
|
3326
|
-
}
|
3327
|
-
if (auto* result = r.value_if_ready()) {
|
3328
|
-
if (grpc_call_trace.enabled()) {
|
3329
|
-
gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s",
|
3330
|
-
DebugTag().c_str(),
|
3331
|
-
recv_close_op_cancel_state_.ToString().c_str(),
|
3332
|
-
(*result)->DebugString().c_str());
|
3333
|
-
}
|
3334
|
-
if (recv_close_op_cancel_state_.CompleteCall(
|
3335
|
-
(*result)->get(GrpcStatusFromWire()).value_or(false))) {
|
3336
|
-
FinishOpOnCompletion(&recv_close_completion_,
|
3337
|
-
PendingOp::kReceiveCloseOnServer);
|
3338
|
-
}
|
3339
|
-
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3340
|
-
if (channelz_node != nullptr) {
|
3341
|
-
if ((*result)
|
3342
|
-
->get(GrpcStatusMetadata())
|
3343
|
-
.value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
|
3344
|
-
channelz_node->RecordCallSucceeded();
|
3345
|
-
} else {
|
3346
|
-
channelz_node->RecordCallFailed();
|
3347
|
-
}
|
3348
|
-
}
|
3349
|
-
if (send_status_from_server_completion_.has_value()) {
|
3350
|
-
FinishOpOnCompletion(&send_status_from_server_completion_,
|
3351
|
-
PendingOp::kSendStatusFromServer);
|
3352
|
-
}
|
3353
|
-
CancelSendMessage();
|
3354
|
-
CancelRecvMessage();
|
3355
|
-
set_completed();
|
3356
|
-
promise_ = ArenaPromise<ServerMetadataHandle>();
|
3204
|
+
channelz::ServerNode* channelz_node = server_->channelz_node();
|
3205
|
+
if (channelz_node != nullptr) {
|
3206
|
+
if (result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN) ==
|
3207
|
+
GRPC_STATUS_OK) {
|
3208
|
+
channelz_node->RecordCallSucceeded();
|
3209
|
+
} else {
|
3210
|
+
channelz_node->RecordCallFailed();
|
3357
3211
|
}
|
3358
3212
|
}
|
3213
|
+
set_completed();
|
3214
|
+
ResetDeadline();
|
3215
|
+
PropagateCancellationToChildren();
|
3359
3216
|
}
|
3360
3217
|
|
3361
3218
|
grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
@@ -3396,56 +3253,92 @@ grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
|
|
3396
3253
|
|
3397
3254
|
void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
|
3398
3255
|
const Completion& completion) {
|
3256
|
+
Party::BulkSpawner spawner(this);
|
3399
3257
|
for (size_t op_idx = 0; op_idx < nops; op_idx++) {
|
3400
3258
|
const grpc_op& op = ops[op_idx];
|
3401
3259
|
switch (op.op) {
|
3402
3260
|
case GRPC_OP_SEND_INITIAL_METADATA: {
|
3403
|
-
|
3404
|
-
|
3405
|
-
|
3406
|
-
|
3407
|
-
|
3408
|
-
|
3409
|
-
|
3410
|
-
if (grpc_call_trace.enabled()) {
|
3411
|
-
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3412
|
-
DebugTag().c_str());
|
3413
|
-
}
|
3414
|
-
auto* pipe = absl::get<PipeSender<ServerMetadataHandle>*>(
|
3415
|
-
send_initial_metadata_state_);
|
3416
|
-
send_initial_metadata_state_ = pipe->Push(std::move(metadata));
|
3261
|
+
auto metadata = arena()->MakePooled<ServerMetadata>(arena());
|
3262
|
+
PrepareOutgoingInitialMetadata(op, *metadata);
|
3263
|
+
CToMetadata(op.data.send_initial_metadata.metadata,
|
3264
|
+
op.data.send_initial_metadata.count, metadata.get());
|
3265
|
+
if (grpc_call_trace.enabled()) {
|
3266
|
+
gpr_log(GPR_INFO, "%s[call] Send initial metadata",
|
3267
|
+
DebugTag().c_str());
|
3417
3268
|
}
|
3269
|
+
QueueSend();
|
3270
|
+
spawner.Spawn(
|
3271
|
+
"call_send_initial_metadata",
|
3272
|
+
[this, metadata = std::move(metadata)]() mutable {
|
3273
|
+
EnactSend();
|
3274
|
+
return server_initial_metadata_->Push(std::move(metadata));
|
3275
|
+
},
|
3276
|
+
[this,
|
3277
|
+
completion = AddOpToCompletion(
|
3278
|
+
completion, PendingOp::kSendInitialMetadata)](bool r) mutable {
|
3279
|
+
if (!r) FailCompletion(completion);
|
3280
|
+
FinishOpOnCompletion(&completion,
|
3281
|
+
PendingOp::kSendInitialMetadata);
|
3282
|
+
});
|
3418
3283
|
} break;
|
3419
3284
|
case GRPC_OP_SEND_MESSAGE:
|
3420
|
-
StartSendMessage(op, completion, server_to_client_messages_);
|
3285
|
+
StartSendMessage(op, completion, server_to_client_messages_, spawner);
|
3421
3286
|
break;
|
3422
3287
|
case GRPC_OP_RECV_MESSAGE:
|
3423
|
-
|
3288
|
+
if (cancelled_.load(std::memory_order_relaxed)) {
|
3289
|
+
FailCompletion(completion);
|
3290
|
+
break;
|
3291
|
+
}
|
3292
|
+
StartRecvMessage(
|
3293
|
+
op, completion, []() { return []() { return Empty{}; }; },
|
3294
|
+
client_to_server_messages_, spawner);
|
3424
3295
|
break;
|
3425
|
-
case GRPC_OP_SEND_STATUS_FROM_SERVER:
|
3426
|
-
|
3296
|
+
case GRPC_OP_SEND_STATUS_FROM_SERVER: {
|
3297
|
+
auto metadata = arena()->MakePooled<ServerMetadata>(arena());
|
3427
3298
|
CToMetadata(op.data.send_status_from_server.trailing_metadata,
|
3428
3299
|
op.data.send_status_from_server.trailing_metadata_count,
|
3429
|
-
|
3430
|
-
|
3431
|
-
|
3300
|
+
metadata.get());
|
3301
|
+
metadata->Set(GrpcStatusMetadata(),
|
3302
|
+
op.data.send_status_from_server.status);
|
3432
3303
|
if (auto* details = op.data.send_status_from_server.status_details) {
|
3433
|
-
|
3434
|
-
Slice(CSliceRef(*details)));
|
3304
|
+
metadata->Set(GrpcMessageMetadata(), Slice(CSliceRef(*details)));
|
3435
3305
|
}
|
3436
|
-
|
3437
|
-
|
3438
|
-
|
3306
|
+
spawner.Spawn(
|
3307
|
+
"call_send_status_from_server",
|
3308
|
+
[this, metadata = std::move(metadata)]() mutable {
|
3309
|
+
bool r = true;
|
3310
|
+
if (send_trailing_metadata_.is_set()) {
|
3311
|
+
r = false;
|
3312
|
+
} else {
|
3313
|
+
send_trailing_metadata_.Set(std::move(metadata));
|
3314
|
+
}
|
3315
|
+
return Map(WaitForSendingStarted(), [this, r](Empty) {
|
3316
|
+
server_initial_metadata_->Close();
|
3317
|
+
server_to_client_messages_->Close();
|
3318
|
+
return r;
|
3319
|
+
});
|
3320
|
+
},
|
3321
|
+
[this, completion = AddOpToCompletion(
|
3322
|
+
completion, PendingOp::kSendStatusFromServer)](
|
3323
|
+
bool ok) mutable {
|
3324
|
+
if (!ok) FailCompletion(completion);
|
3325
|
+
FinishOpOnCompletion(&completion,
|
3326
|
+
PendingOp::kSendStatusFromServer);
|
3327
|
+
});
|
3328
|
+
} break;
|
3439
3329
|
case GRPC_OP_RECV_CLOSE_ON_SERVER:
|
3440
3330
|
if (grpc_call_trace.enabled()) {
|
3441
3331
|
gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
|
3442
3332
|
DebugTag().c_str(),
|
3443
3333
|
recv_close_op_cancel_state_.ToString().c_str());
|
3444
3334
|
}
|
3445
|
-
|
3335
|
+
ForceCompletionSuccess(completion);
|
3336
|
+
recv_close_completion_ =
|
3337
|
+
AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
|
3338
|
+
if (recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
|
3446
3339
|
op.data.recv_close_on_server.cancelled)) {
|
3447
|
-
recv_close_completion_
|
3448
|
-
|
3340
|
+
FinishOpOnCompletion(&recv_close_completion_,
|
3341
|
+
PendingOp::kReceiveCloseOnServer);
|
3449
3342
|
}
|
3450
3343
|
break;
|
3451
3344
|
case GRPC_OP_RECV_STATUS_ON_CLIENT:
|
@@ -3460,8 +3353,6 @@ grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
|
3460
3353
|
size_t nops,
|
3461
3354
|
void* notify_tag,
|
3462
3355
|
bool is_notify_tag_closure) {
|
3463
|
-
MutexLock lock(mu());
|
3464
|
-
ScopedContext activity_context(this);
|
3465
3356
|
if (nops == 0) {
|
3466
3357
|
EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
|
3467
3358
|
return GRPC_CALL_OK;
|
@@ -3473,18 +3364,30 @@ grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
|
|
3473
3364
|
Completion completion =
|
3474
3365
|
StartCompletion(notify_tag, is_notify_tag_closure, ops);
|
3475
3366
|
CommitBatch(ops, nops, completion);
|
3476
|
-
Update();
|
3477
3367
|
FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
|
3478
3368
|
return GRPC_CALL_OK;
|
3479
3369
|
}
|
3480
3370
|
|
3481
|
-
void ServerPromiseBasedCall::
|
3482
|
-
|
3483
|
-
|
3484
|
-
|
3485
|
-
|
3371
|
+
void ServerPromiseBasedCall::CancelWithError(absl::Status error) {
|
3372
|
+
cancelled_.store(true, std::memory_order_relaxed);
|
3373
|
+
Spawn(
|
3374
|
+
"cancel_with_error",
|
3375
|
+
[this, error = std::move(error)]() {
|
3376
|
+
if (!send_trailing_metadata_.is_set()) {
|
3377
|
+
auto md = ServerMetadataFromStatus(error);
|
3378
|
+
md->Set(GrpcCallWasCancelled(), true);
|
3379
|
+
send_trailing_metadata_.Set(std::move(md));
|
3380
|
+
}
|
3381
|
+
if (server_to_client_messages_ != nullptr) {
|
3382
|
+
server_to_client_messages_->Close();
|
3383
|
+
}
|
3384
|
+
if (server_initial_metadata_ != nullptr) {
|
3385
|
+
server_initial_metadata_->Close();
|
3386
|
+
}
|
3387
|
+
return Empty{};
|
3388
|
+
},
|
3389
|
+
[](Empty) {});
|
3486
3390
|
}
|
3487
|
-
|
3488
3391
|
#endif
|
3489
3392
|
|
3490
3393
|
#ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
|
@@ -3493,24 +3396,19 @@ ServerCallContext::MakeTopOfServerCallPromise(
|
|
3493
3396
|
CallArgs call_args, grpc_completion_queue* cq,
|
3494
3397
|
grpc_metadata_array* publish_initial_metadata,
|
3495
3398
|
absl::FunctionRef<void(grpc_call* call)> publish) {
|
3496
|
-
call_->
|
3497
|
-
call_->SetCompletionQueueLocked(cq);
|
3399
|
+
call_->SetCompletionQueue(cq);
|
3498
3400
|
call_->server_to_client_messages_ = call_args.server_to_client_messages;
|
3499
3401
|
call_->client_to_server_messages_ = call_args.client_to_server_messages;
|
3500
|
-
call_->
|
3501
|
-
call_->incoming_compression_algorithm_ =
|
3502
|
-
call_args.client_initial_metadata->get(GrpcEncodingMetadata())
|
3503
|
-
.value_or(GRPC_COMPRESS_NONE);
|
3402
|
+
call_->server_initial_metadata_ = call_args.server_initial_metadata;
|
3504
3403
|
call_->client_initial_metadata_ =
|
3505
3404
|
std::move(call_args.client_initial_metadata);
|
3405
|
+
call_->ProcessIncomingInitialMetadata(*call_->client_initial_metadata_);
|
3506
3406
|
PublishMetadataArray(call_->client_initial_metadata_.get(),
|
3507
3407
|
publish_initial_metadata);
|
3508
3408
|
call_->ExternalRef();
|
3509
3409
|
publish(call_->c_ptr());
|
3510
|
-
return
|
3511
|
-
|
3512
|
-
return call_->PollTopOfCall();
|
3513
|
-
};
|
3410
|
+
return Seq(call_->server_to_client_messages_->AwaitClosed(),
|
3411
|
+
call_->send_trailing_metadata_.Wait());
|
3514
3412
|
}
|
3515
3413
|
#else
|
3516
3414
|
ArenaPromise<ServerMetadataHandle>
|
@@ -3613,7 +3511,9 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
|
|
3613
3511
|
}
|
3614
3512
|
|
3615
3513
|
uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) {
|
3616
|
-
return grpc_core::Call::FromC(call)
|
3514
|
+
return grpc_core::Call::FromC(call)
|
3515
|
+
->encodings_accepted_by_peer()
|
3516
|
+
.ToLegacyBitmask();
|
3617
3517
|
}
|
3618
3518
|
|
3619
3519
|
grpc_core::Arena* grpc_call_get_arena(grpc_call* call) {
|
@@ -3662,7 +3562,9 @@ uint8_t grpc_call_is_client(grpc_call* call) {
|
|
3662
3562
|
|
3663
3563
|
grpc_compression_algorithm grpc_call_compression_for_level(
|
3664
3564
|
grpc_call* call, grpc_compression_level level) {
|
3665
|
-
return grpc_core::Call::FromC(call)
|
3565
|
+
return grpc_core::Call::FromC(call)
|
3566
|
+
->encodings_accepted_by_peer()
|
3567
|
+
.CompressionAlgorithmForLevel(level);
|
3666
3568
|
}
|
3667
3569
|
|
3668
3570
|
bool grpc_call_is_trailers_only(const grpc_call* call) {
|