grpc 1.53.0.pre2 → 1.54.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of grpc might be problematic. Click here for more details.

Files changed (685) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +80 -66
  3. data/include/grpc/event_engine/event_engine.h +30 -14
  4. data/include/grpc/grpc_security.h +4 -0
  5. data/include/grpc/impl/grpc_types.h +11 -2
  6. data/include/grpc/support/port_platform.h +4 -4
  7. data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +11 -0
  8. data/src/core/ext/filters/client_channel/backend_metric.cc +6 -0
  9. data/src/core/ext/filters/client_channel/backup_poller.cc +2 -11
  10. data/src/core/ext/filters/client_channel/backup_poller.h +0 -3
  11. data/src/core/ext/filters/client_channel/client_channel.cc +848 -813
  12. data/src/core/ext/filters/client_channel/client_channel.h +131 -173
  13. data/src/core/ext/filters/client_channel/client_channel_internal.h +114 -0
  14. data/src/core/ext/filters/client_channel/config_selector.h +4 -3
  15. data/src/core/ext/filters/client_channel/http_proxy.cc +1 -1
  16. data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +6 -1
  17. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +17 -18
  18. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +134 -151
  19. data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +1 -15
  20. data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +14 -10
  21. data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc +68 -30
  22. data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +13 -5
  23. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +8 -1
  24. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +2 -5
  25. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +2 -2
  26. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +30 -38
  27. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +4 -4
  28. data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +20 -26
  29. data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +31 -179
  30. data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +1 -2
  31. data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +1 -2
  32. data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +4 -2
  33. data/src/core/ext/filters/client_channel/retry_filter.cc +95 -102
  34. data/src/core/ext/filters/client_channel/subchannel.cc +2 -4
  35. data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +26 -27
  36. data/src/core/ext/filters/client_channel/subchannel_stream_client.h +8 -5
  37. data/src/core/ext/filters/http/client/http_client_filter.cc +3 -3
  38. data/src/core/ext/filters/http/http_filters_plugin.cc +1 -12
  39. data/src/core/ext/filters/http/message_compress/compression_filter.cc +27 -11
  40. data/src/core/ext/filters/message_size/message_size_filter.cc +141 -224
  41. data/src/core/ext/filters/message_size/message_size_filter.h +48 -3
  42. data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +7 -6
  43. data/src/core/ext/gcp/metadata_query.cc +142 -0
  44. data/src/core/ext/gcp/metadata_query.h +82 -0
  45. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +70 -55
  46. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +149 -60
  47. data/src/core/ext/transport/chttp2/transport/flow_control.cc +5 -2
  48. data/src/core/ext/transport/chttp2/transport/flow_control.h +2 -1
  49. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +4 -1
  50. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +42 -23
  51. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +5 -3
  52. data/src/core/ext/transport/chttp2/transport/internal.h +18 -3
  53. data/src/core/ext/transport/chttp2/transport/parsing.cc +9 -2
  54. data/src/core/ext/transport/chttp2/transport/writing.cc +10 -5
  55. data/src/core/ext/transport/inproc/inproc_transport.cc +20 -14
  56. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +5 -3
  57. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +22 -0
  58. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +5 -3
  59. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +22 -0
  60. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +23 -5
  61. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +94 -3
  62. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +23 -2
  63. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +120 -0
  64. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +6 -3
  65. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +22 -0
  66. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +24 -6
  67. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +111 -12
  68. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +9 -7
  69. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +27 -9
  70. data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c +0 -1
  71. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +11 -7
  72. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +56 -12
  73. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c +5 -3
  74. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +24 -0
  75. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c +5 -3
  76. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +24 -0
  77. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +13 -2
  78. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +49 -0
  79. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +24 -9
  80. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +66 -12
  81. data/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +191 -187
  82. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c +139 -136
  83. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.c +31 -15
  84. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/proxy_protocol.upbdefs.h +5 -0
  85. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c +12 -9
  86. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.h +15 -0
  87. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c +54 -45
  88. data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c +135 -119
  89. data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.h +5 -0
  90. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +100 -97
  91. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c +15 -18
  92. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +272 -264
  93. data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +117 -117
  94. data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/ads.upbdefs.c +5 -5
  95. data/src/core/ext/upbdefs-generated/envoy/service/load_stats/v3/lrs.upbdefs.c +5 -5
  96. data/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c +5 -5
  97. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.c +12 -9
  98. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/http_inputs.upbdefs.h +5 -0
  99. data/src/core/ext/xds/xds_channel_stack_modifier.cc +1 -2
  100. data/src/core/ext/xds/xds_client_stats.cc +29 -15
  101. data/src/core/ext/xds/xds_client_stats.h +24 -20
  102. data/src/core/ext/xds/xds_endpoint.cc +5 -2
  103. data/src/core/ext/xds/xds_endpoint.h +9 -1
  104. data/src/core/ext/xds/xds_http_rbac_filter.cc +1 -1
  105. data/src/core/ext/xds/xds_lb_policy_registry.cc +13 -0
  106. data/src/core/ext/xds/xds_transport_grpc.cc +1 -1
  107. data/src/core/{ext/filters/client_channel/resolver/dns/dns_resolver_selection.h → lib/backoff/random_early_detection.cc} +14 -12
  108. data/src/core/lib/backoff/random_early_detection.h +59 -0
  109. data/src/core/lib/channel/call_finalization.h +1 -1
  110. data/src/core/lib/channel/call_tracer.cc +51 -0
  111. data/src/core/lib/channel/call_tracer.h +101 -38
  112. data/src/core/lib/channel/connected_channel.cc +483 -1050
  113. data/src/core/lib/channel/context.h +8 -1
  114. data/src/core/lib/channel/promise_based_filter.cc +106 -42
  115. data/src/core/lib/channel/promise_based_filter.h +27 -13
  116. data/src/core/lib/channel/server_call_tracer_filter.cc +110 -0
  117. data/src/core/lib/config/config_vars.cc +151 -0
  118. data/src/core/lib/config/config_vars.h +127 -0
  119. data/src/core/lib/config/config_vars_non_generated.cc +51 -0
  120. data/src/core/lib/config/load_config.cc +66 -0
  121. data/src/core/lib/config/load_config.h +49 -0
  122. data/src/core/lib/debug/trace.cc +5 -6
  123. data/src/core/lib/debug/trace.h +0 -5
  124. data/src/core/lib/event_engine/event_engine.cc +37 -2
  125. data/src/core/lib/event_engine/handle_containers.h +7 -22
  126. data/src/core/lib/event_engine/memory_allocator_factory.h +47 -0
  127. data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +0 -4
  128. data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +3 -9
  129. data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +48 -15
  130. data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +8 -8
  131. data/src/core/lib/event_engine/posix_engine/posix_engine.cc +6 -5
  132. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +6 -3
  133. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +27 -18
  134. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +0 -3
  135. data/src/core/lib/event_engine/resolved_address.cc +2 -1
  136. data/src/core/lib/event_engine/windows/win_socket.cc +0 -1
  137. data/src/core/lib/event_engine/windows/windows_endpoint.cc +129 -82
  138. data/src/core/lib/event_engine/windows/windows_endpoint.h +21 -5
  139. data/src/core/lib/event_engine/windows/windows_engine.cc +39 -18
  140. data/src/core/lib/event_engine/windows/windows_engine.h +2 -1
  141. data/src/core/lib/event_engine/windows/windows_listener.cc +370 -0
  142. data/src/core/lib/event_engine/windows/windows_listener.h +155 -0
  143. data/src/core/lib/experiments/config.cc +3 -10
  144. data/src/core/lib/experiments/experiments.cc +7 -0
  145. data/src/core/lib/experiments/experiments.h +9 -1
  146. data/src/core/lib/gpr/log.cc +15 -28
  147. data/src/core/lib/gprpp/fork.cc +8 -14
  148. data/src/core/lib/gprpp/orphanable.h +4 -3
  149. data/src/core/lib/gprpp/per_cpu.h +9 -3
  150. data/src/core/lib/gprpp/{thd_posix.cc → posix/thd.cc} +49 -37
  151. data/src/core/lib/gprpp/ref_counted.h +33 -34
  152. data/src/core/lib/gprpp/thd.h +16 -0
  153. data/src/core/lib/gprpp/time.cc +1 -0
  154. data/src/core/lib/gprpp/time.h +4 -4
  155. data/src/core/lib/gprpp/{thd_windows.cc → windows/thd.cc} +2 -2
  156. data/src/core/lib/iomgr/call_combiner.h +2 -2
  157. data/src/core/lib/iomgr/endpoint_cfstream.cc +4 -2
  158. data/src/core/lib/iomgr/endpoint_pair.h +2 -2
  159. data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -2
  160. data/src/core/lib/iomgr/endpoint_pair_windows.cc +1 -1
  161. data/src/core/lib/iomgr/ev_posix.cc +13 -53
  162. data/src/core/lib/iomgr/ev_posix.h +0 -3
  163. data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +103 -76
  164. data/src/core/lib/iomgr/iomgr.cc +4 -8
  165. data/src/core/lib/iomgr/iomgr_windows.cc +8 -2
  166. data/src/core/lib/iomgr/pollset_set_windows.cc +9 -9
  167. data/src/core/lib/iomgr/pollset_windows.cc +1 -1
  168. data/src/core/lib/iomgr/socket_utils_common_posix.cc +16 -3
  169. data/src/core/lib/iomgr/tcp_client_windows.cc +2 -2
  170. data/src/core/lib/iomgr/tcp_posix.cc +0 -1
  171. data/src/core/lib/iomgr/tcp_server_posix.cc +5 -16
  172. data/src/core/lib/iomgr/tcp_server_windows.cc +176 -9
  173. data/src/core/lib/iomgr/tcp_windows.cc +12 -8
  174. data/src/core/lib/load_balancing/lb_policy.cc +9 -13
  175. data/src/core/lib/load_balancing/lb_policy.h +4 -2
  176. data/src/core/lib/promise/activity.cc +22 -6
  177. data/src/core/lib/promise/activity.h +61 -24
  178. data/src/core/lib/promise/cancel_callback.h +77 -0
  179. data/src/core/lib/promise/detail/basic_seq.h +1 -1
  180. data/src/core/lib/promise/detail/promise_factory.h +4 -0
  181. data/src/core/lib/promise/for_each.h +176 -0
  182. data/src/core/lib/promise/if.h +9 -0
  183. data/src/core/lib/promise/interceptor_list.h +23 -2
  184. data/src/core/lib/promise/latch.h +89 -3
  185. data/src/core/lib/promise/loop.h +13 -9
  186. data/src/core/lib/promise/map.h +7 -0
  187. data/src/core/lib/promise/party.cc +286 -0
  188. data/src/core/lib/promise/party.h +499 -0
  189. data/src/core/lib/promise/pipe.h +197 -57
  190. data/src/core/lib/promise/poll.h +48 -0
  191. data/src/core/lib/promise/promise.h +2 -2
  192. data/src/core/lib/resource_quota/arena.cc +19 -3
  193. data/src/core/lib/resource_quota/arena.h +119 -5
  194. data/src/core/lib/resource_quota/memory_quota.cc +1 -1
  195. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +12 -35
  196. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +1 -0
  197. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +0 -59
  198. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +10 -5
  199. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +1 -1
  200. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +13 -0
  201. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +2 -0
  202. data/src/core/lib/security/security_connector/load_system_roots_supported.cc +5 -9
  203. data/src/core/lib/security/security_connector/ssl_utils.cc +11 -25
  204. data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +12 -0
  205. data/src/core/lib/security/transport/secure_endpoint.cc +4 -2
  206. data/src/core/lib/security/transport/server_auth_filter.cc +20 -2
  207. data/src/core/lib/slice/slice.cc +1 -1
  208. data/src/core/lib/surface/builtins.cc +2 -0
  209. data/src/core/lib/surface/call.cc +926 -1024
  210. data/src/core/lib/surface/call.h +10 -0
  211. data/src/core/lib/surface/lame_client.cc +1 -0
  212. data/src/core/lib/surface/version.cc +2 -2
  213. data/src/core/lib/transport/batch_builder.cc +179 -0
  214. data/src/core/lib/transport/batch_builder.h +468 -0
  215. data/src/core/lib/transport/bdp_estimator.cc +7 -7
  216. data/src/core/lib/transport/bdp_estimator.h +10 -6
  217. data/src/core/lib/transport/custom_metadata.h +30 -0
  218. data/src/core/lib/transport/metadata_batch.cc +9 -6
  219. data/src/core/lib/transport/metadata_batch.h +58 -16
  220. data/src/core/lib/transport/parsed_metadata.h +3 -3
  221. data/src/core/lib/transport/timeout_encoding.cc +6 -1
  222. data/src/core/lib/transport/transport.cc +30 -2
  223. data/src/core/lib/transport/transport.h +70 -14
  224. data/src/core/lib/transport/transport_impl.h +7 -0
  225. data/src/core/lib/transport/transport_op_string.cc +52 -42
  226. data/src/core/plugin_registry/grpc_plugin_registry.cc +2 -2
  227. data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +1 -0
  228. data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +21 -4
  229. data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +5 -0
  230. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +1 -1
  231. data/src/core/tsi/ssl_transport_security.cc +4 -2
  232. data/src/ruby/lib/grpc/version.rb +1 -1
  233. data/third_party/abseil-cpp/absl/base/config.h +1 -1
  234. data/third_party/abseil-cpp/absl/flags/commandlineflag.cc +34 -0
  235. data/third_party/abseil-cpp/absl/flags/commandlineflag.h +200 -0
  236. data/third_party/abseil-cpp/absl/flags/config.h +68 -0
  237. data/third_party/abseil-cpp/absl/flags/declare.h +73 -0
  238. data/third_party/abseil-cpp/absl/flags/flag.cc +38 -0
  239. data/third_party/abseil-cpp/absl/flags/flag.h +310 -0
  240. data/{src/core/lib/gprpp/global_config_custom.h → third_party/abseil-cpp/absl/flags/internal/commandlineflag.cc} +11 -14
  241. data/third_party/abseil-cpp/absl/flags/internal/commandlineflag.h +68 -0
  242. data/third_party/abseil-cpp/absl/flags/internal/flag.cc +615 -0
  243. data/third_party/abseil-cpp/absl/flags/internal/flag.h +800 -0
  244. data/third_party/abseil-cpp/absl/flags/internal/flag_msvc.inc +116 -0
  245. data/third_party/abseil-cpp/absl/flags/internal/path_util.h +62 -0
  246. data/third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.cc +65 -0
  247. data/third_party/abseil-cpp/absl/flags/internal/private_handle_accessor.h +61 -0
  248. data/third_party/abseil-cpp/absl/flags/internal/program_name.cc +60 -0
  249. data/third_party/abseil-cpp/absl/flags/internal/program_name.h +50 -0
  250. data/third_party/abseil-cpp/absl/flags/internal/registry.h +97 -0
  251. data/third_party/abseil-cpp/absl/flags/internal/sequence_lock.h +187 -0
  252. data/third_party/abseil-cpp/absl/flags/marshalling.cc +241 -0
  253. data/third_party/abseil-cpp/absl/flags/marshalling.h +356 -0
  254. data/third_party/abseil-cpp/absl/flags/reflection.cc +354 -0
  255. data/third_party/abseil-cpp/absl/flags/reflection.h +90 -0
  256. data/third_party/abseil-cpp/absl/flags/usage_config.cc +165 -0
  257. data/third_party/abseil-cpp/absl/flags/usage_config.h +135 -0
  258. data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +12 -8
  259. data/third_party/boringssl-with-bazel/err_data.c +728 -712
  260. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_bitstr.c +177 -177
  261. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_bool.c +28 -55
  262. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_d2i_fp.c +21 -23
  263. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_dup.c +20 -23
  264. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_gentm.c +66 -185
  265. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_i2d_fp.c +18 -21
  266. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_int.c +356 -311
  267. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_mbstr.c +174 -194
  268. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_object.c +146 -210
  269. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_octet.c +6 -9
  270. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_strex.c +346 -526
  271. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_strnid.c +110 -131
  272. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_time.c +130 -116
  273. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_type.c +93 -60
  274. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_utctm.c +93 -181
  275. data/third_party/boringssl-with-bazel/src/crypto/asn1/asn1_lib.c +242 -305
  276. data/third_party/boringssl-with-bazel/src/crypto/asn1/asn1_par.c +41 -18
  277. data/third_party/boringssl-with-bazel/src/crypto/asn1/asn_pack.c +30 -33
  278. data/third_party/boringssl-with-bazel/src/crypto/asn1/f_int.c +36 -33
  279. data/third_party/boringssl-with-bazel/src/crypto/asn1/f_string.c +29 -26
  280. data/third_party/boringssl-with-bazel/src/crypto/asn1/internal.h +133 -88
  281. data/third_party/boringssl-with-bazel/src/crypto/asn1/posix_time.c +230 -0
  282. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_dec.c +791 -791
  283. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_enc.c +526 -526
  284. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_fre.c +114 -135
  285. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_new.c +201 -207
  286. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_typ.c +21 -26
  287. data/third_party/boringssl-with-bazel/src/crypto/asn1/tasn_utl.c +55 -68
  288. data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +2 -4
  289. data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +11 -7
  290. data/third_party/boringssl-with-bazel/src/crypto/bio/bio_mem.c +4 -4
  291. data/third_party/boringssl-with-bazel/src/crypto/bio/connect.c +15 -9
  292. data/third_party/boringssl-with-bazel/src/crypto/bio/fd.c +4 -4
  293. data/third_party/boringssl-with-bazel/src/crypto/bio/file.c +17 -10
  294. data/third_party/boringssl-with-bazel/src/crypto/bio/pair.c +1 -3
  295. data/third_party/boringssl-with-bazel/src/crypto/bio/printf.c +0 -13
  296. data/third_party/boringssl-with-bazel/src/crypto/bio/socket.c +3 -6
  297. data/third_party/boringssl-with-bazel/src/crypto/bio/socket_helper.c +2 -0
  298. data/third_party/boringssl-with-bazel/src/crypto/blake2/blake2.c +9 -5
  299. data/third_party/boringssl-with-bazel/src/crypto/bn_extra/convert.c +10 -23
  300. data/third_party/boringssl-with-bazel/src/crypto/buf/buf.c +2 -6
  301. data/third_party/boringssl-with-bazel/src/crypto/bytestring/asn1_compat.c +2 -1
  302. data/third_party/boringssl-with-bazel/src/crypto/bytestring/ber.c +29 -28
  303. data/third_party/boringssl-with-bazel/src/crypto/bytestring/cbb.c +161 -201
  304. data/third_party/boringssl-with-bazel/src/crypto/bytestring/cbs.c +254 -39
  305. data/third_party/boringssl-with-bazel/src/crypto/bytestring/internal.h +2 -2
  306. data/third_party/boringssl-with-bazel/src/crypto/chacha/chacha.c +0 -2
  307. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/derive_key.c +4 -4
  308. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesctrhmac.c +9 -8
  309. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_aesgcmsiv.c +37 -75
  310. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_chacha20poly1305.c +8 -10
  311. data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/cipher → cipher_extra}/e_des.c +100 -78
  312. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_null.c +1 -0
  313. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc2.c +1 -0
  314. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc4.c +2 -0
  315. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_tls.c +6 -12
  316. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/internal.h +14 -11
  317. data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +6 -10
  318. data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -1
  319. data/third_party/boringssl-with-bazel/src/crypto/conf/internal.h +12 -0
  320. data/third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_apple.c +74 -0
  321. data/third_party/boringssl-with-bazel/src/crypto/cpu_aarch64_freebsd.c +62 -0
  322. data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-fuchsia.c → cpu_aarch64_fuchsia.c} +8 -7
  323. data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-linux.c → cpu_aarch64_linux.c} +6 -4
  324. data/third_party/boringssl-with-bazel/src/crypto/{cpu-aarch64-win.c → cpu_aarch64_win.c} +4 -4
  325. data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm.c → cpu_arm.c} +1 -1
  326. data/third_party/boringssl-with-bazel/src/crypto/cpu_arm_freebsd.c +55 -0
  327. data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm-linux.c → cpu_arm_linux.c} +11 -90
  328. data/third_party/boringssl-with-bazel/src/crypto/{cpu-arm-linux.h → cpu_arm_linux.h} +0 -38
  329. data/third_party/boringssl-with-bazel/src/crypto/{cpu-intel.c → cpu_intel.c} +1 -2
  330. data/third_party/boringssl-with-bazel/src/crypto/crypto.c +25 -20
  331. data/third_party/boringssl-with-bazel/src/crypto/curve25519/curve25519.c +16 -27
  332. data/third_party/boringssl-with-bazel/src/crypto/curve25519/spake25519.c +17 -32
  333. data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/des → des}/des.c +232 -232
  334. data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/des → des}/internal.h +1 -1
  335. data/third_party/boringssl-with-bazel/src/crypto/dh_extra/dh_asn1.c +1 -0
  336. data/third_party/boringssl-with-bazel/src/crypto/dh_extra/params.c +232 -29
  337. data/third_party/boringssl-with-bazel/src/crypto/digest_extra/digest_extra.c +0 -3
  338. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +39 -16
  339. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +37 -7
  340. data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +3 -3
  341. data/third_party/boringssl-with-bazel/src/crypto/ec_extra/ec_asn1.c +11 -36
  342. data/third_party/boringssl-with-bazel/src/crypto/ec_extra/hash_to_curve.c +214 -99
  343. data/third_party/boringssl-with-bazel/src/crypto/ec_extra/internal.h +21 -5
  344. data/third_party/boringssl-with-bazel/src/crypto/ecdsa_extra/ecdsa_asn1.c +2 -4
  345. data/third_party/boringssl-with-bazel/src/crypto/err/err.c +83 -60
  346. data/third_party/boringssl-with-bazel/src/crypto/evp/evp.c +46 -12
  347. data/third_party/boringssl-with-bazel/src/crypto/evp/evp_asn1.c +3 -3
  348. data/third_party/boringssl-with-bazel/src/crypto/evp/evp_ctx.c +25 -23
  349. data/third_party/boringssl-with-bazel/src/crypto/evp/internal.h +43 -9
  350. data/third_party/boringssl-with-bazel/src/crypto/evp/p_dsa_asn1.c +75 -44
  351. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec.c +19 -25
  352. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec_asn1.c +96 -45
  353. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519.c +7 -8
  354. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519_asn1.c +26 -23
  355. data/third_party/boringssl-with-bazel/src/crypto/evp/p_hkdf.c +233 -0
  356. data/third_party/boringssl-with-bazel/src/crypto/evp/p_rsa.c +5 -5
  357. data/third_party/boringssl-with-bazel/src/crypto/evp/p_rsa_asn1.c +42 -25
  358. data/third_party/boringssl-with-bazel/src/crypto/evp/p_x25519.c +4 -5
  359. data/third_party/boringssl-with-bazel/src/crypto/evp/p_x25519_asn1.c +35 -47
  360. data/third_party/boringssl-with-bazel/src/crypto/evp/print.c +135 -244
  361. data/third_party/boringssl-with-bazel/src/crypto/evp/scrypt.c +2 -4
  362. data/third_party/boringssl-with-bazel/src/crypto/evp/sign.c +15 -10
  363. data/third_party/boringssl-with-bazel/src/crypto/ex_data.c +29 -15
  364. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes.c +0 -2
  365. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes_nohw.c +13 -14
  366. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/internal.h +3 -13
  367. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/key_wrap.c +13 -7
  368. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/mode_wrappers.c +9 -7
  369. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +35 -27
  370. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bn.c +16 -26
  371. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bytes.c +88 -60
  372. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/cmp.c +4 -3
  373. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/ctx.c +0 -2
  374. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div.c +1 -1
  375. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div_extra.c +1 -1
  376. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +99 -113
  377. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd.c +0 -1
  378. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd_extra.c +5 -3
  379. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/generic.c +112 -168
  380. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +86 -31
  381. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery.c +11 -6
  382. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery_inv.c +4 -5
  383. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/mul.c +4 -5
  384. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/prime.c +13 -0
  385. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/random.c +13 -5
  386. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/rsaz_exp.c +19 -108
  387. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/rsaz_exp.h +19 -15
  388. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/shift.c +15 -16
  389. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/sqrt.c +22 -21
  390. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/aead.c +3 -0
  391. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/cipher.c +79 -19
  392. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/e_aes.c +102 -99
  393. data/third_party/boringssl-with-bazel/src/crypto/{cipher_extra → fipsmodule/cipher}/e_aesccm.c +52 -46
  394. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/internal.h +39 -0
  395. data/third_party/boringssl-with-bazel/src/crypto/{cmac → fipsmodule/cmac}/cmac.c +55 -11
  396. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/check.c +2 -3
  397. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/dh.c +21 -6
  398. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/internal.h +56 -0
  399. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/digest/digest.c +5 -3
  400. data/third_party/boringssl-with-bazel/src/crypto/{evp → fipsmodule/digestsign}/digestsign.c +51 -15
  401. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec.c +25 -25
  402. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_key.c +91 -17
  403. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_montgomery.c +5 -5
  404. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/internal.h +34 -12
  405. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/oct.c +54 -23
  406. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p224-64.c +44 -60
  407. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64-table.h → p256-nistz-table.h} +1 -1
  408. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64.c → p256-nistz.c} +60 -53
  409. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256-x86_64.h → p256-nistz.h} +5 -13
  410. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p256.c +48 -36
  411. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/scalar.c +2 -8
  412. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/simple.c +2 -7
  413. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/simple_mul.c +2 -3
  414. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/wnaf.c +0 -1
  415. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdh/ecdh.c +8 -0
  416. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/ecdsa.c +42 -14
  417. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/internal.h +6 -0
  418. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/hmac/hmac.c +52 -24
  419. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/cbc.c +9 -15
  420. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/cfb.c +1 -4
  421. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/ctr.c +2 -4
  422. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/gcm.c +71 -43
  423. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/internal.h +14 -16
  424. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/ofb.c +1 -4
  425. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/ctrdrbg.c +31 -13
  426. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.c +16 -8
  427. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.h +3 -2
  428. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/getrandom_fillin.h +2 -2
  429. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/internal.h +9 -38
  430. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/rand.c +73 -59
  431. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/urandom.c +11 -45
  432. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/blinding.c +0 -1
  433. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/internal.h +22 -0
  434. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/padding.c +63 -52
  435. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa.c +107 -62
  436. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa_impl.c +58 -31
  437. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/fips.c +41 -0
  438. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/self_check.c +523 -422
  439. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/internal.h +89 -0
  440. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/service_indicator.c +334 -0
  441. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +3 -12
  442. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1.c +2 -0
  443. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha256.c +12 -8
  444. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +14 -12
  445. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/tls/kdf.c +19 -6
  446. data/third_party/boringssl-with-bazel/src/crypto/hpke/hpke.c +32 -14
  447. data/third_party/boringssl-with-bazel/src/crypto/hrss/hrss.c +65 -29
  448. data/third_party/boringssl-with-bazel/src/crypto/internal.h +373 -18
  449. data/third_party/boringssl-with-bazel/src/crypto/kyber/internal.h +61 -0
  450. data/third_party/boringssl-with-bazel/src/crypto/kyber/keccak.c +205 -0
  451. data/third_party/boringssl-with-bazel/src/crypto/lhash/internal.h +13 -1
  452. data/third_party/boringssl-with-bazel/src/crypto/mem.c +220 -13
  453. data/third_party/boringssl-with-bazel/src/crypto/obj/obj.c +19 -7
  454. data/third_party/boringssl-with-bazel/src/crypto/obj/obj_dat.h +13 -1
  455. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_all.c +81 -90
  456. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_info.c +150 -245
  457. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_lib.c +629 -613
  458. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_oth.c +17 -17
  459. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pk8.c +142 -149
  460. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pkey.c +99 -131
  461. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_x509.c +0 -1
  462. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_xaux.c +0 -1
  463. data/third_party/boringssl-with-bazel/src/crypto/pkcs7/pkcs7_x509.c +0 -1
  464. data/third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8.c +0 -3
  465. data/third_party/boringssl-with-bazel/src/crypto/pkcs8/pkcs8_x509.c +36 -66
  466. data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305.c +31 -38
  467. data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_arm.c +2 -1
  468. data/third_party/boringssl-with-bazel/src/crypto/poly1305/poly1305_vec.c +18 -31
  469. data/third_party/boringssl-with-bazel/src/crypto/pool/internal.h +1 -0
  470. data/third_party/boringssl-with-bazel/src/crypto/pool/pool.c +8 -1
  471. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/passive.c +129 -5
  472. data/third_party/boringssl-with-bazel/src/crypto/refcount_c11.c +0 -2
  473. data/third_party/boringssl-with-bazel/src/crypto/refcount_lock.c +3 -4
  474. data/third_party/boringssl-with-bazel/src/crypto/siphash/siphash.c +8 -11
  475. data/third_party/boringssl-with-bazel/src/crypto/stack/stack.c +61 -27
  476. data/third_party/boringssl-with-bazel/src/crypto/thread_pthread.c +10 -13
  477. data/third_party/boringssl-with-bazel/src/crypto/thread_win.c +10 -13
  478. data/third_party/boringssl-with-bazel/src/crypto/trust_token/internal.h +66 -34
  479. data/third_party/boringssl-with-bazel/src/crypto/trust_token/pmbtoken.c +190 -77
  480. data/third_party/boringssl-with-bazel/src/crypto/trust_token/trust_token.c +81 -284
  481. data/third_party/boringssl-with-bazel/src/crypto/trust_token/voprf.c +109 -42
  482. data/third_party/boringssl-with-bazel/src/crypto/x509/a_digest.c +22 -24
  483. data/third_party/boringssl-with-bazel/src/crypto/x509/a_sign.c +54 -55
  484. data/third_party/boringssl-with-bazel/src/crypto/x509/a_verify.c +32 -34
  485. data/third_party/boringssl-with-bazel/src/crypto/x509/algorithm.c +32 -16
  486. data/third_party/boringssl-with-bazel/src/crypto/x509/asn1_gen.c +465 -704
  487. data/third_party/boringssl-with-bazel/src/crypto/x509/by_dir.c +284 -331
  488. data/third_party/boringssl-with-bazel/src/crypto/x509/by_file.c +183 -178
  489. data/third_party/boringssl-with-bazel/src/crypto/x509/i2d_pr.c +11 -15
  490. data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +67 -50
  491. data/third_party/boringssl-with-bazel/src/crypto/x509/name_print.c +153 -150
  492. data/third_party/boringssl-with-bazel/src/crypto/x509/policy.c +786 -0
  493. data/third_party/boringssl-with-bazel/src/crypto/x509/rsa_pss.c +95 -102
  494. data/third_party/boringssl-with-bazel/src/crypto/x509/t_crl.c +72 -57
  495. data/third_party/boringssl-with-bazel/src/crypto/x509/t_req.c +12 -10
  496. data/third_party/boringssl-with-bazel/src/crypto/x509/t_x509.c +227 -252
  497. data/third_party/boringssl-with-bazel/src/crypto/x509/t_x509a.c +52 -47
  498. data/third_party/boringssl-with-bazel/src/crypto/x509/x509.c +3 -4
  499. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_att.c +230 -224
  500. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_cmp.c +161 -327
  501. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_d2.c +37 -33
  502. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_def.c +14 -31
  503. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_ext.c +55 -85
  504. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +534 -618
  505. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_obj.c +129 -122
  506. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_req.c +116 -182
  507. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_set.c +132 -132
  508. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_trs.c +181 -202
  509. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_txt.c +64 -79
  510. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_v3.c +175 -160
  511. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +1865 -2050
  512. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vpm.c +433 -462
  513. data/third_party/boringssl-with-bazel/src/crypto/x509/x509cset.c +156 -163
  514. data/third_party/boringssl-with-bazel/src/crypto/x509/x509name.c +267 -263
  515. data/third_party/boringssl-with-bazel/src/crypto/x509/x509rset.c +40 -15
  516. data/third_party/boringssl-with-bazel/src/crypto/x509/x509spki.c +59 -63
  517. data/third_party/boringssl-with-bazel/src/crypto/x509/x_algor.c +63 -67
  518. data/third_party/boringssl-with-bazel/src/crypto/x509/x_all.c +114 -144
  519. data/third_party/boringssl-with-bazel/src/crypto/x509/x_attrib.c +25 -26
  520. data/third_party/boringssl-with-bazel/src/crypto/x509/x_crl.c +326 -415
  521. data/third_party/boringssl-with-bazel/src/crypto/x509/x_exten.c +8 -7
  522. data/third_party/boringssl-with-bazel/src/crypto/x509/x_info.c +30 -28
  523. data/third_party/boringssl-with-bazel/src/crypto/x509/x_name.c +354 -370
  524. data/third_party/boringssl-with-bazel/src/crypto/x509/x_pkey.c +37 -32
  525. data/third_party/boringssl-with-bazel/src/crypto/x509/x_pubkey.c +116 -119
  526. data/third_party/boringssl-with-bazel/src/crypto/x509/x_req.c +36 -26
  527. data/third_party/boringssl-with-bazel/src/crypto/x509/x_sig.c +3 -4
  528. data/third_party/boringssl-with-bazel/src/crypto/x509/x_spki.c +10 -13
  529. data/third_party/boringssl-with-bazel/src/crypto/x509/x_val.c +3 -4
  530. data/third_party/boringssl-with-bazel/src/crypto/x509/x_x509.c +419 -261
  531. data/third_party/boringssl-with-bazel/src/crypto/x509/x_x509a.c +113 -105
  532. data/third_party/boringssl-with-bazel/src/crypto/x509v3/ext_dat.h +11 -15
  533. data/third_party/boringssl-with-bazel/src/crypto/x509v3/internal.h +78 -170
  534. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_akey.c +126 -131
  535. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_akeya.c +3 -4
  536. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_alt.c +465 -469
  537. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_bcons.c +56 -54
  538. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_bitst.c +46 -49
  539. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_conf.c +309 -346
  540. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_cpols.c +341 -365
  541. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_crld.c +429 -393
  542. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_enum.c +29 -24
  543. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_extku.c +65 -59
  544. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_genn.c +125 -121
  545. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ia5.c +43 -42
  546. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_info.c +122 -125
  547. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_int.c +50 -20
  548. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_lib.c +247 -253
  549. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ncons.c +386 -389
  550. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_ocsp.c +45 -32
  551. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pcons.c +57 -54
  552. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pmaps.c +63 -67
  553. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_prn.c +143 -136
  554. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_purp.c +664 -707
  555. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_skey.c +83 -75
  556. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_utl.c +1062 -1146
  557. data/third_party/boringssl-with-bazel/src/include/openssl/aead.h +8 -4
  558. data/third_party/boringssl-with-bazel/src/include/openssl/arm_arch.h +28 -48
  559. data/third_party/boringssl-with-bazel/src/include/openssl/asn1.h +211 -187
  560. data/third_party/boringssl-with-bazel/src/include/openssl/asn1t.h +26 -78
  561. data/third_party/boringssl-with-bazel/src/include/openssl/base.h +19 -14
  562. data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +21 -2
  563. data/third_party/boringssl-with-bazel/src/include/openssl/bn.h +49 -17
  564. data/third_party/boringssl-with-bazel/src/include/openssl/bytestring.h +99 -29
  565. data/third_party/boringssl-with-bazel/src/include/openssl/cipher.h +49 -60
  566. data/third_party/boringssl-with-bazel/src/include/openssl/conf.h +2 -15
  567. data/third_party/boringssl-with-bazel/src/include/openssl/cpu.h +16 -200
  568. data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +34 -0
  569. data/third_party/boringssl-with-bazel/src/include/openssl/ctrdrbg.h +82 -0
  570. data/third_party/boringssl-with-bazel/src/include/openssl/dh.h +32 -30
  571. data/third_party/boringssl-with-bazel/src/include/openssl/digest.h +7 -0
  572. data/third_party/boringssl-with-bazel/src/include/openssl/dsa.h +4 -0
  573. data/third_party/boringssl-with-bazel/src/include/openssl/ec.h +48 -5
  574. data/third_party/boringssl-with-bazel/src/include/openssl/ec_key.h +37 -8
  575. data/third_party/boringssl-with-bazel/src/include/openssl/ecdsa.h +1 -0
  576. data/third_party/boringssl-with-bazel/src/include/openssl/err.h +33 -5
  577. data/third_party/boringssl-with-bazel/src/include/openssl/evp.h +22 -30
  578. data/third_party/boringssl-with-bazel/src/include/openssl/ex_data.h +1 -1
  579. data/third_party/boringssl-with-bazel/src/include/openssl/hmac.h +7 -0
  580. data/third_party/boringssl-with-bazel/src/include/openssl/hpke.h +41 -16
  581. data/third_party/boringssl-with-bazel/src/include/openssl/kdf.h +91 -0
  582. data/third_party/boringssl-with-bazel/src/include/openssl/mem.h +74 -8
  583. data/third_party/boringssl-with-bazel/src/include/openssl/nid.h +13 -0
  584. data/third_party/boringssl-with-bazel/src/include/openssl/opensslconf.h +1 -0
  585. data/third_party/boringssl-with-bazel/src/include/openssl/pem.h +11 -15
  586. data/third_party/boringssl-with-bazel/src/include/openssl/pkcs8.h +8 -0
  587. data/third_party/boringssl-with-bazel/src/include/openssl/rand.h +12 -1
  588. data/third_party/boringssl-with-bazel/src/include/openssl/rsa.h +7 -4
  589. data/third_party/boringssl-with-bazel/src/include/openssl/service_indicator.h +96 -0
  590. data/third_party/boringssl-with-bazel/src/include/openssl/span.h +13 -21
  591. data/third_party/boringssl-with-bazel/src/include/openssl/ssl.h +139 -75
  592. data/third_party/boringssl-with-bazel/src/include/openssl/ssl3.h +1 -6
  593. data/third_party/boringssl-with-bazel/src/include/openssl/stack.h +384 -286
  594. data/third_party/boringssl-with-bazel/src/include/openssl/thread.h +5 -6
  595. data/third_party/boringssl-with-bazel/src/include/openssl/time.h +41 -0
  596. data/third_party/boringssl-with-bazel/src/include/openssl/tls1.h +18 -7
  597. data/third_party/boringssl-with-bazel/src/include/openssl/trust_token.h +49 -23
  598. data/third_party/boringssl-with-bazel/src/include/openssl/type_check.h +0 -11
  599. data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +1592 -1074
  600. data/third_party/boringssl-with-bazel/src/include/openssl/x509v3.h +202 -205
  601. data/third_party/boringssl-with-bazel/src/ssl/bio_ssl.cc +2 -2
  602. data/third_party/boringssl-with-bazel/src/ssl/d1_both.cc +6 -13
  603. data/third_party/boringssl-with-bazel/src/ssl/d1_pkt.cc +17 -18
  604. data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +4 -5
  605. data/third_party/boringssl-with-bazel/src/ssl/dtls_record.cc +25 -33
  606. data/third_party/boringssl-with-bazel/src/ssl/encrypted_client_hello.cc +34 -20
  607. data/third_party/boringssl-with-bazel/src/ssl/extensions.cc +65 -34
  608. data/third_party/boringssl-with-bazel/src/ssl/handoff.cc +198 -54
  609. data/third_party/boringssl-with-bazel/src/ssl/handshake.cc +5 -5
  610. data/third_party/boringssl-with-bazel/src/ssl/handshake_client.cc +32 -28
  611. data/third_party/boringssl-with-bazel/src/ssl/handshake_server.cc +76 -44
  612. data/third_party/boringssl-with-bazel/src/ssl/internal.h +130 -98
  613. data/third_party/boringssl-with-bazel/src/ssl/s3_both.cc +27 -11
  614. data/third_party/boringssl-with-bazel/src/ssl/s3_lib.cc +2 -2
  615. data/third_party/boringssl-with-bazel/src/ssl/s3_pkt.cc +91 -75
  616. data/third_party/boringssl-with-bazel/src/ssl/ssl_aead_ctx.cc +8 -10
  617. data/third_party/boringssl-with-bazel/src/ssl/ssl_asn1.cc +39 -65
  618. data/third_party/boringssl-with-bazel/src/ssl/ssl_buffer.cc +1 -0
  619. data/third_party/boringssl-with-bazel/src/ssl/ssl_cert.cc +5 -9
  620. data/third_party/boringssl-with-bazel/src/ssl/ssl_cipher.cc +30 -33
  621. data/third_party/boringssl-with-bazel/src/ssl/ssl_file.cc +77 -100
  622. data/third_party/boringssl-with-bazel/src/ssl/ssl_key_share.cc +120 -107
  623. data/third_party/boringssl-with-bazel/src/ssl/ssl_lib.cc +164 -30
  624. data/third_party/boringssl-with-bazel/src/ssl/ssl_privkey.cc +150 -60
  625. data/third_party/boringssl-with-bazel/src/ssl/ssl_session.cc +22 -11
  626. data/third_party/boringssl-with-bazel/src/ssl/ssl_x509.cc +22 -6
  627. data/third_party/boringssl-with-bazel/src/ssl/t1_enc.cc +15 -13
  628. data/third_party/boringssl-with-bazel/src/ssl/tls13_both.cc +5 -43
  629. data/third_party/boringssl-with-bazel/src/ssl/tls13_client.cc +7 -4
  630. data/third_party/boringssl-with-bazel/src/ssl/tls13_enc.cc +2 -2
  631. data/third_party/boringssl-with-bazel/src/ssl/tls13_server.cc +22 -34
  632. data/third_party/boringssl-with-bazel/src/ssl/tls_method.cc +2 -2
  633. data/third_party/boringssl-with-bazel/src/ssl/tls_record.cc +16 -98
  634. data/third_party/boringssl-with-bazel/src/third_party/fiat/curve25519_32.h +1241 -657
  635. data/third_party/boringssl-with-bazel/src/third_party/fiat/curve25519_64.h +751 -398
  636. data/third_party/boringssl-with-bazel/src/third_party/fiat/p256_32.h +3551 -1938
  637. data/third_party/boringssl-with-bazel/src/third_party/fiat/p256_64.h +1272 -487
  638. metadata +107 -72
  639. data/src/core/ext/filters/client_channel/lb_call_state_internal.h +0 -39
  640. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +0 -30
  641. data/src/core/lib/gprpp/global_config.h +0 -93
  642. data/src/core/lib/gprpp/global_config_env.cc +0 -140
  643. data/src/core/lib/gprpp/global_config_env.h +0 -133
  644. data/src/core/lib/gprpp/global_config_generic.h +0 -40
  645. data/src/core/lib/promise/intra_activity_waiter.h +0 -55
  646. data/src/core/lib/security/security_connector/ssl_utils_config.cc +0 -32
  647. data/src/core/lib/security/security_connector/ssl_utils_config.h +0 -29
  648. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_enum.c +0 -195
  649. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_print.c +0 -83
  650. data/third_party/boringssl-with-bazel/src/crypto/asn1/a_utf8.c +0 -236
  651. data/third_party/boringssl-with-bazel/src/crypto/asn1/charmap.h +0 -15
  652. data/third_party/boringssl-with-bazel/src/crypto/asn1/time_support.c +0 -206
  653. data/third_party/boringssl-with-bazel/src/crypto/cpu-ppc64le.c +0 -38
  654. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1-altivec.c +0 -361
  655. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_cache.c +0 -287
  656. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_data.c +0 -132
  657. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_lib.c +0 -155
  658. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_map.c +0 -131
  659. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_node.c +0 -189
  660. data/third_party/boringssl-with-bazel/src/crypto/x509v3/pcy_tree.c +0 -843
  661. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pci.c +0 -289
  662. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_pcia.c +0 -57
  663. /data/src/core/lib/gpr/{log_android.cc → android/log.cc} +0 -0
  664. /data/src/core/lib/gpr/{cpu_iphone.cc → iphone/cpu.cc} +0 -0
  665. /data/src/core/lib/gpr/{cpu_linux.cc → linux/cpu.cc} +0 -0
  666. /data/src/core/lib/gpr/{log_linux.cc → linux/log.cc} +0 -0
  667. /data/src/core/lib/gpr/{tmpfile_msys.cc → msys/tmpfile.cc} +0 -0
  668. /data/src/core/lib/gpr/{cpu_posix.cc → posix/cpu.cc} +0 -0
  669. /data/src/core/lib/gpr/{log_posix.cc → posix/log.cc} +0 -0
  670. /data/src/core/lib/gpr/{string_posix.cc → posix/string.cc} +0 -0
  671. /data/src/core/lib/gpr/{sync_posix.cc → posix/sync.cc} +0 -0
  672. /data/src/core/lib/gpr/{time_posix.cc → posix/time.cc} +0 -0
  673. /data/src/core/lib/gpr/{tmpfile_posix.cc → posix/tmpfile.cc} +0 -0
  674. /data/src/core/lib/gpr/{cpu_windows.cc → windows/cpu.cc} +0 -0
  675. /data/src/core/lib/gpr/{log_windows.cc → windows/log.cc} +0 -0
  676. /data/src/core/lib/gpr/{string_windows.cc → windows/string.cc} +0 -0
  677. /data/src/core/lib/gpr/{string_util_windows.cc → windows/string_util.cc} +0 -0
  678. /data/src/core/lib/gpr/{sync_windows.cc → windows/sync.cc} +0 -0
  679. /data/src/core/lib/gpr/{time_windows.cc → windows/time.cc} +0 -0
  680. /data/src/core/lib/gpr/{tmpfile_windows.cc → windows/tmpfile.cc} +0 -0
  681. /data/src/core/lib/gprpp/{env_linux.cc → linux/env.cc} +0 -0
  682. /data/src/core/lib/gprpp/{env_posix.cc → posix/env.cc} +0 -0
  683. /data/src/core/lib/gprpp/{stat_posix.cc → posix/stat.cc} +0 -0
  684. /data/src/core/lib/gprpp/{env_windows.cc → windows/env.cc} +0 -0
  685. /data/src/core/lib/gprpp/{stat_windows.cc → windows/stat.cc} +0 -0
@@ -23,6 +23,7 @@
23
23
  #include <inttypes.h>
24
24
  #include <limits.h>
25
25
  #include <stdlib.h>
26
+ #include <string.h>
26
27
 
27
28
  #include <algorithm>
28
29
  #include <atomic>
@@ -35,14 +36,12 @@
35
36
  #include <vector>
36
37
 
37
38
  #include "absl/base/thread_annotations.h"
38
- #include "absl/cleanup/cleanup.h"
39
39
  #include "absl/meta/type_traits.h"
40
40
  #include "absl/status/status.h"
41
41
  #include "absl/strings/str_cat.h"
42
42
  #include "absl/strings/str_format.h"
43
43
  #include "absl/strings/str_join.h"
44
44
  #include "absl/strings/string_view.h"
45
- #include "absl/types/variant.h"
46
45
 
47
46
  #include <grpc/byte_buffer.h>
48
47
  #include <grpc/compression.h>
@@ -59,6 +58,8 @@
59
58
  #include <grpc/support/time.h>
60
59
 
61
60
  #include "src/core/lib/channel/call_finalization.h"
61
+ #include "src/core/lib/channel/call_tracer.h"
62
+ #include "src/core/lib/channel/channel_args.h"
62
63
  #include "src/core/lib/channel/channel_stack.h"
63
64
  #include "src/core/lib/channel/channelz.h"
64
65
  #include "src/core/lib/channel/context.h"
@@ -85,8 +86,13 @@
85
86
  #include "src/core/lib/promise/arena_promise.h"
86
87
  #include "src/core/lib/promise/context.h"
87
88
  #include "src/core/lib/promise/detail/basic_seq.h"
89
+ #include "src/core/lib/promise/latch.h"
90
+ #include "src/core/lib/promise/map.h"
91
+ #include "src/core/lib/promise/party.h"
88
92
  #include "src/core/lib/promise/pipe.h"
89
93
  #include "src/core/lib/promise/poll.h"
94
+ #include "src/core/lib/promise/race.h"
95
+ #include "src/core/lib/promise/seq.h"
90
96
  #include "src/core/lib/resource_quota/arena.h"
91
97
  #include "src/core/lib/slice/slice_buffer.h"
92
98
  #include "src/core/lib/slice/slice_internal.h"
@@ -96,6 +102,7 @@
96
102
  #include "src/core/lib/surface/completion_queue.h"
97
103
  #include "src/core/lib/surface/server.h"
98
104
  #include "src/core/lib/surface/validate_metadata.h"
105
+ #include "src/core/lib/transport/batch_builder.h"
99
106
  #include "src/core/lib/transport/error_utils.h"
100
107
  #include "src/core/lib/transport/metadata_batch.h"
101
108
  #include "src/core/lib/transport/transport.h"
@@ -134,18 +141,18 @@ class Call : public CppImplOf<Call, grpc_call> {
134
141
  virtual void InternalRef(const char* reason) = 0;
135
142
  virtual void InternalUnref(const char* reason) = 0;
136
143
 
137
- virtual grpc_compression_algorithm test_only_compression_algorithm() = 0;
138
- virtual uint32_t test_only_message_flags() = 0;
139
- virtual uint32_t test_only_encodings_accepted_by_peer() = 0;
140
- virtual grpc_compression_algorithm compression_for_level(
141
- grpc_compression_level level) = 0;
144
+ grpc_compression_algorithm test_only_compression_algorithm() {
145
+ return incoming_compression_algorithm_;
146
+ }
147
+ uint32_t test_only_message_flags() { return test_only_last_message_flags_; }
148
+ CompressionAlgorithmSet encodings_accepted_by_peer() {
149
+ return encodings_accepted_by_peer_;
150
+ }
142
151
 
143
152
  // This should return nullptr for the promise stack (and alternative means
144
153
  // for that functionality be invented)
145
154
  virtual grpc_call_stack* call_stack() = 0;
146
155
 
147
- gpr_atm* peer_string_atm_ptr() { return &peer_string_; }
148
-
149
156
  protected:
150
157
  // The maximum number of concurrent batches possible.
151
158
  // Based upon the maximum number of individually queueable ops in the batch
@@ -203,7 +210,37 @@ class Call : public CppImplOf<Call, grpc_call> {
203
210
  send_deadline_ = send_deadline;
204
211
  }
205
212
 
206
- void ClearPeerString() { gpr_atm_rel_store(&peer_string_, 0); }
213
+ Slice GetPeerString() const {
214
+ MutexLock lock(&peer_mu_);
215
+ return peer_string_.Ref();
216
+ }
217
+
218
+ void SetPeerString(Slice peer_string) {
219
+ MutexLock lock(&peer_mu_);
220
+ peer_string_ = std::move(peer_string);
221
+ }
222
+
223
+ void ClearPeerString() { SetPeerString(Slice(grpc_empty_slice())); }
224
+
225
+ // TODO(ctiller): cancel_func is for cancellation of the call - filter stack
226
+ // holds no mutexes here, promise stack does, and so locking is different.
227
+ // Remove this and cancel directly once promise conversion is done.
228
+ void ProcessIncomingInitialMetadata(grpc_metadata_batch& md);
229
+ // Fixup outgoing metadata before sending - adds compression, protects
230
+ // internal headers against external modification.
231
+ void PrepareOutgoingInitialMetadata(const grpc_op& op,
232
+ grpc_metadata_batch& md);
233
+ void NoteLastMessageFlags(uint32_t flags) {
234
+ test_only_last_message_flags_ = flags;
235
+ }
236
+ grpc_compression_algorithm incoming_compression_algorithm() const {
237
+ return incoming_compression_algorithm_;
238
+ }
239
+
240
+ void HandleCompressionAlgorithmDisabled(
241
+ grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
242
+ void HandleCompressionAlgorithmNotAccepted(
243
+ grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
207
244
 
208
245
  private:
209
246
  RefCountedPtr<Channel> channel_;
@@ -214,8 +251,18 @@ class Call : public CppImplOf<Call, grpc_call> {
214
251
  const bool is_client_;
215
252
  // flag indicating that cancellation is inherited
216
253
  bool cancellation_is_inherited_ = false;
217
- // A char* indicating the peer name.
218
- gpr_atm peer_string_ = 0;
254
+ // Compression algorithm for *incoming* data
255
+ grpc_compression_algorithm incoming_compression_algorithm_ =
256
+ GRPC_COMPRESS_NONE;
257
+ // Supported encodings (compression algorithms), a bitset.
258
+ // Always support no compression.
259
+ CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
260
+ uint32_t test_only_last_message_flags_ = 0;
261
+ // Peer name is protected by a mutex because it can be accessed by the
262
+ // application at the same moment as it is being set by the completion
263
+ // of the recv_initial_metadata op. The mutex should be mostly uncontended.
264
+ mutable Mutex peer_mu_;
265
+ Slice peer_string_;
219
266
  };
220
267
 
221
268
  Call::ParentCall* Call::GetOrCreateParentCall() {
@@ -310,9 +357,13 @@ void Call::MaybeUnpublishFromParent() {
310
357
  void Call::CancelWithStatus(grpc_status_code status, const char* description) {
311
358
  // copying 'description' is needed to ensure the grpc_call_cancel_with_status
312
359
  // guarantee that can be short-lived.
360
+ // TODO(ctiller): change to
361
+ // absl::Status(static_cast<absl::StatusCode>(status), description)
362
+ // (ie remove the set_int, set_str).
313
363
  CancelWithError(grpc_error_set_int(
314
- grpc_error_set_str(GRPC_ERROR_CREATE(description),
315
- StatusStrProperty::kGrpcMessage, description),
364
+ grpc_error_set_str(
365
+ absl::Status(static_cast<absl::StatusCode>(status), description),
366
+ StatusStrProperty::kGrpcMessage, description),
316
367
  StatusIntProperty::kRpcStatus, status));
317
368
  }
318
369
 
@@ -337,9 +388,16 @@ void Call::PropagateCancellationToChildren() {
337
388
  }
338
389
 
339
390
  char* Call::GetPeer() {
340
- char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
341
- if (peer_string != nullptr) return gpr_strdup(peer_string);
342
- peer_string = grpc_channel_get_target(channel_->c_ptr());
391
+ Slice peer_slice = GetPeerString();
392
+ if (!peer_slice.empty()) {
393
+ absl::string_view peer_string_view = peer_slice.as_string_view();
394
+ char* peer_string =
395
+ static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
396
+ memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
397
+ peer_string[peer_string_view.size()] = '\0';
398
+ return peer_string;
399
+ }
400
+ char* peer_string = grpc_channel_get_target(channel_->c_ptr());
343
401
  if (peer_string != nullptr) return peer_string;
344
402
  return gpr_strdup("unknown");
345
403
  }
@@ -352,6 +410,92 @@ void Call::DeleteThis() {
352
410
  arena->Destroy();
353
411
  }
354
412
 
413
+ void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
414
+ grpc_metadata_batch& md) {
415
+ // TODO(juanlishen): If the user has already specified a compression
416
+ // algorithm by setting the initial metadata with key of
417
+ // GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
418
+ // with the compression algorithm mapped from compression level.
419
+ // process compression level
420
+ grpc_compression_level effective_compression_level = GRPC_COMPRESS_LEVEL_NONE;
421
+ bool level_set = false;
422
+ if (op.data.send_initial_metadata.maybe_compression_level.is_set) {
423
+ effective_compression_level =
424
+ op.data.send_initial_metadata.maybe_compression_level.level;
425
+ level_set = true;
426
+ } else {
427
+ const grpc_compression_options copts = channel()->compression_options();
428
+ if (copts.default_level.is_set) {
429
+ level_set = true;
430
+ effective_compression_level = copts.default_level.level;
431
+ }
432
+ }
433
+ // Currently, only server side supports compression level setting.
434
+ if (level_set && !is_client()) {
435
+ const grpc_compression_algorithm calgo =
436
+ encodings_accepted_by_peer().CompressionAlgorithmForLevel(
437
+ effective_compression_level);
438
+ // The following metadata will be checked and removed by the message
439
+ // compression filter. It will be used as the call's compression
440
+ // algorithm.
441
+ md.Set(GrpcInternalEncodingRequest(), calgo);
442
+ }
443
+ // Ignore any te metadata key value pairs specified.
444
+ md.Remove(TeMetadata());
445
+ }
446
+
447
+ void Call::ProcessIncomingInitialMetadata(grpc_metadata_batch& md) {
448
+ Slice* peer_string = md.get_pointer(PeerString());
449
+ if (peer_string != nullptr) SetPeerString(peer_string->Ref());
450
+
451
+ incoming_compression_algorithm_ =
452
+ md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
453
+ encodings_accepted_by_peer_ =
454
+ md.Take(GrpcAcceptEncodingMetadata())
455
+ .value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
456
+
457
+ const grpc_compression_options compression_options =
458
+ channel_->compression_options();
459
+ const grpc_compression_algorithm compression_algorithm =
460
+ incoming_compression_algorithm_;
461
+ if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
462
+ compression_options.enabled_algorithms_bitset)
463
+ .IsSet(compression_algorithm))) {
464
+ // check if algorithm is supported by current channel config
465
+ HandleCompressionAlgorithmDisabled(compression_algorithm);
466
+ }
467
+ // GRPC_COMPRESS_NONE is always set.
468
+ GPR_DEBUG_ASSERT(encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
469
+ if (GPR_UNLIKELY(!encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
470
+ if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
471
+ HandleCompressionAlgorithmNotAccepted(compression_algorithm);
472
+ }
473
+ }
474
+ }
475
+
476
+ void Call::HandleCompressionAlgorithmNotAccepted(
477
+ grpc_compression_algorithm compression_algorithm) {
478
+ const char* algo_name = nullptr;
479
+ grpc_compression_algorithm_name(compression_algorithm, &algo_name);
480
+ gpr_log(GPR_ERROR,
481
+ "Compression algorithm ('%s') not present in the "
482
+ "accepted encodings (%s)",
483
+ algo_name,
484
+ std::string(encodings_accepted_by_peer_.ToString()).c_str());
485
+ }
486
+
487
+ void Call::HandleCompressionAlgorithmDisabled(
488
+ grpc_compression_algorithm compression_algorithm) {
489
+ const char* algo_name = nullptr;
490
+ grpc_compression_algorithm_name(compression_algorithm, &algo_name);
491
+ std::string error_msg =
492
+ absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
493
+ gpr_log(GPR_ERROR, "%s", error_msg.c_str());
494
+ CancelWithError(grpc_error_set_int(absl::UnimplementedError(error_msg),
495
+ StatusIntProperty::kRpcStatus,
496
+ GRPC_STATUS_UNIMPLEMENTED));
497
+ }
498
+
355
499
  ///////////////////////////////////////////////////////////////////////////////
356
500
  // FilterStackCall
357
501
  // To be removed once promise conversion is complete
@@ -410,11 +554,6 @@ class FilterStackCall final : public Call {
410
554
  return context_[elem].value;
411
555
  }
412
556
 
413
- grpc_compression_algorithm compression_for_level(
414
- grpc_compression_level level) override {
415
- return encodings_accepted_by_peer_.CompressionAlgorithmForLevel(level);
416
- }
417
-
418
557
  bool is_trailers_only() const override {
419
558
  bool result = is_trailers_only_;
420
559
  GPR_DEBUG_ASSERT(!result || recv_initial_metadata_.TransportSize() == 0);
@@ -432,18 +571,6 @@ class FilterStackCall final : public Call {
432
571
  return authority_metadata->as_string_view();
433
572
  }
434
573
 
435
- grpc_compression_algorithm test_only_compression_algorithm() override {
436
- return incoming_compression_algorithm_;
437
- }
438
-
439
- uint32_t test_only_message_flags() override {
440
- return test_only_last_message_flags_;
441
- }
442
-
443
- uint32_t test_only_encodings_accepted_by_peer() override {
444
- return encodings_accepted_by_peer_.ToLegacyBitmask();
445
- }
446
-
447
574
  static size_t InitialSizeEstimate() {
448
575
  return sizeof(FilterStackCall) +
449
576
  sizeof(BatchControl) * kMaxConcurrentBatches;
@@ -480,6 +607,7 @@ class FilterStackCall final : public Call {
480
607
  }
481
608
  struct BatchControl {
482
609
  FilterStackCall* call_ = nullptr;
610
+ CallTracerAnnotationInterface* call_tracer_ = nullptr;
483
611
  grpc_transport_stream_op_batch op_;
484
612
  // Share memory for cq_completion and notify_tag as they are never needed
485
613
  // simultaneously. Each byte used in this data structure count as six bytes
@@ -509,12 +637,31 @@ class FilterStackCall final : public Call {
509
637
  }
510
638
  bool completed_batch_step(PendingOp op) {
511
639
  auto mask = PendingOpMask(op);
640
+ // Acquire call tracer before ops_pending_.fetch_sub to avoid races with
641
+ // call_ being set to nullptr in PostCompletion method. Store the
642
+ // call_tracer_ and call_ variables locally as well because they could be
643
+ // modified by another thread after the fetch_sub operation.
644
+ CallTracerAnnotationInterface* call_tracer = call_tracer_;
645
+ FilterStackCall* call = call_;
646
+ bool is_call_trace_enabled = grpc_call_trace.enabled();
647
+ bool is_call_ops_annotate_enabled =
648
+ (IsTraceRecordCallopsEnabled() && call_tracer != nullptr);
649
+ if (is_call_ops_annotate_enabled) {
650
+ call->InternalRef("Call ops annotate");
651
+ }
512
652
  auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
513
- if (grpc_call_trace.enabled()) {
514
- gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
515
- PendingOpString(mask).c_str(),
516
- PendingOpString(r & ~mask).c_str(),
517
- completion_data_.notify_tag.tag);
653
+ if (is_call_trace_enabled || is_call_ops_annotate_enabled) {
654
+ std::string trace_string = absl::StrFormat(
655
+ "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
656
+ PendingOpString(mask).c_str(), PendingOpString(r & ~mask).c_str(),
657
+ completion_data_.notify_tag.tag);
658
+ if (is_call_trace_enabled) {
659
+ gpr_log(GPR_DEBUG, "%s", trace_string.c_str());
660
+ }
661
+ if (is_call_ops_annotate_enabled) {
662
+ call_tracer->RecordAnnotation(trace_string);
663
+ call->InternalUnref("Call ops annotate");
664
+ }
518
665
  }
519
666
  GPR_ASSERT((r & mask) != 0);
520
667
  return r == mask;
@@ -524,7 +671,6 @@ class FilterStackCall final : public Call {
524
671
  void FinishStep(PendingOp op);
525
672
  void ProcessDataAfterMetadata();
526
673
  void ReceivingStreamReady(grpc_error_handle error);
527
- void ValidateFilteredMetadata();
528
674
  void ReceivingInitialMetadataReady(grpc_error_handle error);
529
675
  void ReceivingTrailingMetadataReady(grpc_error_handle error);
530
676
  void FinishBatch(grpc_error_handle error);
@@ -549,10 +695,6 @@ class FilterStackCall final : public Call {
549
695
  grpc_closure* start_batch_closure);
550
696
  void SetFinalStatus(grpc_error_handle error);
551
697
  BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
552
- void HandleCompressionAlgorithmDisabled(
553
- grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
554
- void HandleCompressionAlgorithmNotAccepted(
555
- grpc_compression_algorithm compression_algorithm) GPR_ATTRIBUTE_NOINLINE;
556
698
  bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
557
699
  bool is_trailing);
558
700
  void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
@@ -596,13 +738,6 @@ class FilterStackCall final : public Call {
596
738
  // completed
597
739
  grpc_call_final_info final_info_;
598
740
 
599
- // Compression algorithm for *incoming* data
600
- grpc_compression_algorithm incoming_compression_algorithm_ =
601
- GRPC_COMPRESS_NONE;
602
- // Supported encodings (compression algorithms), a bitset.
603
- // Always support no compression.
604
- CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
605
-
606
741
  // Contexts for various subsystems (security, tracing, ...).
607
742
  grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
608
743
 
@@ -616,7 +751,6 @@ class FilterStackCall final : public Call {
616
751
  grpc_closure receiving_stream_ready_;
617
752
  grpc_closure receiving_initial_metadata_ready_;
618
753
  grpc_closure receiving_trailing_metadata_ready_;
619
- uint32_t test_only_last_message_flags_ = 0;
620
754
  // Status about operation of call
621
755
  bool sent_server_trailing_metadata_ = false;
622
756
  gpr_atm cancelled_with_error_ = 0;
@@ -703,6 +837,26 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
703
837
  global_stats().IncrementServerCallsCreated();
704
838
  call->final_op_.server.cancelled = nullptr;
705
839
  call->final_op_.server.core_server = args->server;
840
+ // TODO(yashykt): In the future, we want to also enable stats and trace
841
+ // collecting from when the call is created at the transport. The idea is
842
+ // that the transport would create the call tracer and pass it in as part of
843
+ // the metadata.
844
+ auto* server_call_tracer_factory = ServerCallTracerFactory::Get(
845
+ args->server != nullptr ? args->server->channel_args() : ChannelArgs());
846
+ if (server_call_tracer_factory != nullptr) {
847
+ auto* server_call_tracer =
848
+ server_call_tracer_factory->CreateNewServerCallTracer(arena);
849
+ if (server_call_tracer != nullptr) {
850
+ // Note that we are setting both
851
+ // GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
852
+ // GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
853
+ // promise-based world, we would just a single tracer object for each
854
+ // stack (call, subchannel_call, server_call.)
855
+ call->ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
856
+ server_call_tracer, nullptr);
857
+ call->ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
858
+ }
859
+ }
706
860
  }
707
861
 
708
862
  Call* parent = Call::FromC(args->parent);
@@ -1033,11 +1187,7 @@ void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
1033
1187
  }
1034
1188
 
1035
1189
  void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
1036
- incoming_compression_algorithm_ =
1037
- b->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
1038
- encodings_accepted_by_peer_ =
1039
- b->Take(GrpcAcceptEncodingMetadata())
1040
- .value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
1190
+ ProcessIncomingInitialMetadata(*b);
1041
1191
  PublishAppMetadata(b, false);
1042
1192
  }
1043
1193
 
@@ -1052,11 +1202,11 @@ void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
1052
1202
  grpc_status_code status_code = *grpc_status;
1053
1203
  grpc_error_handle error;
1054
1204
  if (status_code != GRPC_STATUS_OK) {
1055
- char* peer = GetPeer();
1205
+ Slice peer = GetPeerString();
1056
1206
  error = grpc_error_set_int(
1057
- GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ", peer)),
1207
+ GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
1208
+ peer.as_string_view())),
1058
1209
  StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
1059
- gpr_free(peer);
1060
1210
  }
1061
1211
  auto grpc_message = b->Take(GrpcMessageMetadata());
1062
1212
  if (grpc_message.has_value()) {
@@ -1133,6 +1283,8 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
1133
1283
  *pslot = bctl;
1134
1284
  }
1135
1285
  bctl->call_ = this;
1286
+ bctl->call_tracer_ = static_cast<CallTracerAnnotationInterface*>(
1287
+ ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
1136
1288
  bctl->op_.payload = &stream_op_payload_;
1137
1289
  return bctl;
1138
1290
  }
@@ -1204,11 +1356,11 @@ void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
1204
1356
  call->receiving_message_ = false;
1205
1357
  FinishStep(PendingOp::kRecvMessage);
1206
1358
  } else {
1207
- call->test_only_last_message_flags_ = call->receiving_stream_flags_;
1359
+ call->NoteLastMessageFlags(call->receiving_stream_flags_);
1208
1360
  if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
1209
- (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
1361
+ (call->incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
1210
1362
  *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
1211
- nullptr, 0, call->incoming_compression_algorithm_);
1363
+ nullptr, 0, call->incoming_compression_algorithm());
1212
1364
  } else {
1213
1365
  *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
1214
1366
  }
@@ -1249,50 +1401,6 @@ void FilterStackCall::BatchControl::ReceivingStreamReady(
1249
1401
  }
1250
1402
  }
1251
1403
 
1252
- void FilterStackCall::HandleCompressionAlgorithmDisabled(
1253
- grpc_compression_algorithm compression_algorithm) {
1254
- const char* algo_name = nullptr;
1255
- grpc_compression_algorithm_name(compression_algorithm, &algo_name);
1256
- std::string error_msg =
1257
- absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
1258
- gpr_log(GPR_ERROR, "%s", error_msg.c_str());
1259
- CancelWithStatus(GRPC_STATUS_UNIMPLEMENTED, error_msg.c_str());
1260
- }
1261
-
1262
- void FilterStackCall::HandleCompressionAlgorithmNotAccepted(
1263
- grpc_compression_algorithm compression_algorithm) {
1264
- const char* algo_name = nullptr;
1265
- grpc_compression_algorithm_name(compression_algorithm, &algo_name);
1266
- gpr_log(GPR_ERROR,
1267
- "Compression algorithm ('%s') not present in the "
1268
- "accepted encodings (%s)",
1269
- algo_name,
1270
- std::string(encodings_accepted_by_peer_.ToString()).c_str());
1271
- }
1272
-
1273
- void FilterStackCall::BatchControl::ValidateFilteredMetadata() {
1274
- FilterStackCall* call = call_;
1275
-
1276
- const grpc_compression_options compression_options =
1277
- call->channel()->compression_options();
1278
- const grpc_compression_algorithm compression_algorithm =
1279
- call->incoming_compression_algorithm_;
1280
- if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
1281
- compression_options.enabled_algorithms_bitset)
1282
- .IsSet(compression_algorithm))) {
1283
- // check if algorithm is supported by current channel config
1284
- call->HandleCompressionAlgorithmDisabled(compression_algorithm);
1285
- }
1286
- // GRPC_COMPRESS_NONE is always set.
1287
- GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
1288
- if (GPR_UNLIKELY(
1289
- !call->encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
1290
- if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
1291
- call->HandleCompressionAlgorithmNotAccepted(compression_algorithm);
1292
- }
1293
- }
1294
- }
1295
-
1296
1404
  void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
1297
1405
  grpc_error_handle error) {
1298
1406
  FilterStackCall* call = call_;
@@ -1303,9 +1411,6 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
1303
1411
  grpc_metadata_batch* md = &call->recv_initial_metadata_;
1304
1412
  call->RecvInitialFilter(md);
1305
1413
 
1306
- // TODO(ctiller): this could be moved into recv_initial_filter now
1307
- ValidateFilteredMetadata();
1308
-
1309
1414
  absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
1310
1415
  if (deadline.has_value() && !call->is_client()) {
1311
1416
  call_->set_send_deadline(*deadline);
@@ -1400,6 +1505,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1400
1505
  grpc_transport_stream_op_batch_payload* stream_op_payload;
1401
1506
  uint32_t seen_ops = 0;
1402
1507
  intptr_t pending_ops = 0;
1508
+ CallTracerAnnotationInterface* call_tracer = nullptr;
1403
1509
 
1404
1510
  for (i = 0; i < nops; i++) {
1405
1511
  if (seen_ops & (1u << ops[i].op)) {
@@ -1454,36 +1560,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1454
1560
  error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1455
1561
  goto done_with_error;
1456
1562
  }
1457
- // TODO(juanlishen): If the user has already specified a compression
1458
- // algorithm by setting the initial metadata with key of
1459
- // GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
1460
- // with the compression algorithm mapped from compression level.
1461
- // process compression level
1462
- grpc_compression_level effective_compression_level =
1463
- GRPC_COMPRESS_LEVEL_NONE;
1464
- bool level_set = false;
1465
- if (op->data.send_initial_metadata.maybe_compression_level.is_set) {
1466
- effective_compression_level =
1467
- op->data.send_initial_metadata.maybe_compression_level.level;
1468
- level_set = true;
1469
- } else {
1470
- const grpc_compression_options copts =
1471
- channel()->compression_options();
1472
- if (copts.default_level.is_set) {
1473
- level_set = true;
1474
- effective_compression_level = copts.default_level.level;
1475
- }
1476
- }
1477
- // Currently, only server side supports compression level setting.
1478
- if (level_set && !is_client()) {
1479
- const grpc_compression_algorithm calgo =
1480
- encodings_accepted_by_peer_.CompressionAlgorithmForLevel(
1481
- effective_compression_level);
1482
- // The following metadata will be checked and removed by the message
1483
- // compression filter. It will be used as the call's compression
1484
- // algorithm.
1485
- send_initial_metadata_.Set(GrpcInternalEncodingRequest(), calgo);
1486
- }
1487
1563
  if (op->data.send_initial_metadata.count > INT_MAX) {
1488
1564
  error = GRPC_CALL_ERROR_INVALID_METADATA;
1489
1565
  goto done_with_error;
@@ -1496,8 +1572,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1496
1572
  error = GRPC_CALL_ERROR_INVALID_METADATA;
1497
1573
  goto done_with_error;
1498
1574
  }
1499
- // Ignore any te metadata key value pairs specified.
1500
- send_initial_metadata_.Remove(TeMetadata());
1575
+ PrepareOutgoingInitialMetadata(*op, send_initial_metadata_);
1501
1576
  // TODO(ctiller): just make these the same variable?
1502
1577
  if (is_client() && send_deadline() != Timestamp::InfFuture()) {
1503
1578
  send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
@@ -1512,10 +1587,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1512
1587
  }
1513
1588
  stream_op_payload->send_initial_metadata.send_initial_metadata =
1514
1589
  &send_initial_metadata_;
1515
- if (is_client()) {
1516
- stream_op_payload->send_initial_metadata.peer_string =
1517
- peer_string_atm_ptr();
1518
- }
1519
1590
  pending_ops |= PendingOpMask(PendingOp::kSends);
1520
1591
  break;
1521
1592
  }
@@ -1664,9 +1735,6 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1664
1735
  if (is_client()) {
1665
1736
  stream_op_payload->recv_initial_metadata.trailing_metadata_available =
1666
1737
  &is_trailers_only_;
1667
- } else {
1668
- stream_op_payload->recv_initial_metadata.peer_string =
1669
- peer_string_atm_ptr();
1670
1738
  }
1671
1739
  pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
1672
1740
  break;
@@ -1798,13 +1866,21 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1798
1866
  stream_op->on_complete = &bctl->finish_batch_;
1799
1867
  }
1800
1868
 
1869
+ call_tracer = static_cast<CallTracerAnnotationInterface*>(
1870
+ ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
1871
+ if ((IsTraceRecordCallopsEnabled() && call_tracer != nullptr)) {
1872
+ call_tracer->RecordAnnotation(absl::StrFormat(
1873
+ "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
1874
+ PendingOpString(pending_ops).c_str(),
1875
+ grpc_transport_stream_op_batch_string(stream_op, true).c_str(),
1876
+ bctl->completion_data_.notify_tag.tag));
1877
+ }
1801
1878
  if (grpc_call_trace.enabled()) {
1802
1879
  gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
1803
1880
  PendingOpString(pending_ops).c_str(),
1804
- grpc_transport_stream_op_batch_string(stream_op).c_str(),
1881
+ grpc_transport_stream_op_batch_string(stream_op, false).c_str(),
1805
1882
  bctl->completion_data_.notify_tag.tag);
1806
1883
  }
1807
-
1808
1884
  ExecuteBatch(stream_op, &bctl->start_batch_);
1809
1885
 
1810
1886
  done:
@@ -1873,8 +1949,7 @@ bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
1873
1949
  // Will be folded into Call once the promise conversion is done
1874
1950
 
1875
1951
  class PromiseBasedCall : public Call,
1876
- public Activity,
1877
- public Wakeable,
1952
+ public Party,
1878
1953
  public grpc_event_engine::experimental::EventEngine::
1879
1954
  Closure /* for deadlines */ {
1880
1955
  public:
@@ -1885,176 +1960,54 @@ class PromiseBasedCall : public Call,
1885
1960
  void (*destroy)(void* value)) override;
1886
1961
  void* ContextGet(grpc_context_index elem) const override;
1887
1962
  void SetCompletionQueue(grpc_completion_queue* cq) override;
1888
- void SetCompletionQueueLocked(grpc_completion_queue* cq)
1889
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
1890
- void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) {
1891
- MutexLock lock(&mu_);
1892
- CancelWithErrorLocked(std::move(error));
1893
- }
1894
- virtual void CancelWithErrorLocked(absl::Status error)
1895
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
1896
- bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) {
1897
- MutexLock lock(&mu_);
1898
- return completed_;
1899
- }
1900
-
1901
- void Orphan() final {
1902
- MutexLock lock(&mu_);
1903
- if (!completed_) {
1904
- CancelWithErrorLocked(absl::CancelledError("Call orphaned"));
1905
- }
1906
- }
1963
+ bool Completed() final { return finished_.IsSet(); }
1907
1964
 
1908
1965
  // Implementation of call refcounting: move this to DualRefCounted once we
1909
1966
  // don't need to maintain FilterStackCall compatibility
1910
- void ExternalRef() final {
1911
- const uint64_t prev_ref_pair =
1912
- refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed);
1913
- if (grpc_call_refcount_trace.enabled()) {
1914
- gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(),
1915
- GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1916
- GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair));
1917
- }
1918
- }
1919
- void ExternalUnref() final {
1920
- const uint64_t prev_ref_pair =
1921
- refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel);
1922
- if (grpc_call_refcount_trace.enabled()) {
1923
- gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(),
1924
- GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1925
- GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1);
1926
- }
1927
- const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
1928
- if (GPR_UNLIKELY(strong_refs == 1)) {
1929
- Orphan();
1930
- }
1931
- // Now drop the weak ref.
1932
- InternalUnref("external_ref");
1933
- }
1967
+ void ExternalRef() final { InternalRef("external"); }
1968
+ void ExternalUnref() final { InternalUnref("external"); }
1934
1969
  void InternalRef(const char* reason) final {
1935
- uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed);
1936
1970
  if (grpc_call_refcount_trace.enabled()) {
1937
- gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason,
1938
- GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n),
1939
- GetWeakRefs(n) + 1);
1971
+ gpr_log(GPR_DEBUG, "INTERNAL_REF:%p:%s", this, reason);
1940
1972
  }
1973
+ Party::IncrementRefCount();
1941
1974
  }
1942
1975
  void InternalUnref(const char* reason) final {
1943
- const uint64_t prev_ref_pair =
1944
- refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel);
1945
1976
  if (grpc_call_refcount_trace.enabled()) {
1946
- gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(),
1947
- reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1948
- GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1);
1949
- }
1950
- if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
1951
- DeleteThis();
1977
+ gpr_log(GPR_DEBUG, "INTERNAL_UNREF:%p:%s", this, reason);
1952
1978
  }
1979
+ Party::Unref();
1953
1980
  }
1954
1981
 
1955
- // Activity methods
1956
- void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
1957
- Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override {
1958
- InternalRef("wakeup");
1959
- // If ASAN is defined, we leverage it to detect dropped Waker objects.
1960
- // Usually Waker must be destroyed or woken up, but (especially with arenas)
1961
- // it's not uncommon to create a Waker and then do neither. In that case it's
1962
- // incredibly fraught to diagnose where the dropped reference to this object was
1963
- // created. Instead, leverage ASAN and create a new object per expected wakeup.
1964
- // Now when we drop such an object ASAN will fail and we'll get a callstack to
1965
- // the creation of the waker in question.
1966
- #if defined(__has_feature)
1967
- #if __has_feature(address_sanitizer)
1968
- #define GRPC_CALL_USES_ASAN_WAKER
1969
- class AsanWaker final : public Wakeable {
1970
- public:
1971
- explicit AsanWaker(PromiseBasedCall* call) : call_(call) {}
1972
-
1973
- void Wakeup(void*) override {
1974
- call_->Wakeup(nullptr);
1975
- delete this;
1976
- }
1977
-
1978
- void Drop(void*) override {
1979
- call_->Drop(nullptr);
1980
- delete this;
1981
- }
1982
-
1983
- std::string ActivityDebugTag(void*) const override {
1984
- return call_->DebugTag();
1985
- }
1986
-
1987
- private:
1988
- PromiseBasedCall* call_;
1989
- };
1990
- return Waker(new AsanWaker(this), nullptr);
1991
- #endif
1992
- #endif
1993
- #ifndef GRPC_CALL_USES_ASAN_WAKER
1994
- return Waker(this, nullptr);
1995
- #endif
1996
- }
1997
- Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
1998
-
1999
- // Wakeable methods
2000
- void Wakeup(void*) override {
2001
- channel()->event_engine()->Run([this] {
2002
- ApplicationCallbackExecCtx app_exec_ctx;
2003
- ExecCtx exec_ctx;
2004
- {
2005
- ScopedContext activity_context(this);
2006
- MutexLock lock(&mu_);
2007
- Update();
2008
- }
2009
- InternalUnref("wakeup");
2010
- });
2011
- }
2012
- void Drop(void*) override { InternalUnref("wakeup"); }
2013
-
2014
1982
  void RunInContext(absl::AnyInvocable<void()> fn) {
2015
- if (Activity::current() == this) {
2016
- fn();
2017
- } else {
2018
- InternalRef("in_context");
2019
- channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable {
2020
- ApplicationCallbackExecCtx app_exec_ctx;
2021
- ExecCtx exec_ctx;
2022
- {
2023
- ScopedContext activity_context(this);
2024
- MutexLock lock(&mu_);
1983
+ Spawn(
1984
+ "run_in_context",
1985
+ [fn = std::move(fn)]() mutable {
2025
1986
  fn();
2026
- Update();
2027
- }
2028
- InternalUnref("in_context");
2029
- });
2030
- }
2031
- }
2032
-
2033
- grpc_compression_algorithm test_only_compression_algorithm() override {
2034
- abort();
2035
- }
2036
- uint32_t test_only_message_flags() override { abort(); }
2037
- uint32_t test_only_encodings_accepted_by_peer() override { abort(); }
2038
- grpc_compression_algorithm compression_for_level(
2039
- grpc_compression_level) override {
2040
- abort();
1987
+ return Empty{};
1988
+ },
1989
+ [](Empty) {});
2041
1990
  }
2042
1991
 
2043
1992
  // This should return nullptr for the promise stack (and alternative means
2044
1993
  // for that functionality be invented)
2045
1994
  grpc_call_stack* call_stack() override { return nullptr; }
2046
1995
 
2047
- void UpdateDeadline(Timestamp deadline);
2048
- void ResetDeadline();
1996
+ void UpdateDeadline(Timestamp deadline) ABSL_LOCKS_EXCLUDED(deadline_mu_);
1997
+ void ResetDeadline() ABSL_LOCKS_EXCLUDED(deadline_mu_);
2049
1998
 
2050
1999
  // Implementation of EventEngine::Closure, called when deadline expires
2051
2000
  void Run() override;
2052
2001
 
2053
2002
  virtual ServerCallContext* server_call_context() { return nullptr; }
2054
2003
 
2004
+ using Call::arena;
2005
+
2055
2006
  protected:
2056
2007
  class ScopedContext
2057
2008
  : public ScopedActivity,
2009
+ public BatchBuilder,
2010
+ public promise_detail::Context<BatchBuilder>,
2058
2011
  public promise_detail::Context<Arena>,
2059
2012
  public promise_detail::Context<grpc_call_context_element>,
2060
2013
  public promise_detail::Context<CallContext>,
@@ -2062,6 +2015,8 @@ class PromiseBasedCall : public Call,
2062
2015
  public:
2063
2016
  explicit ScopedContext(PromiseBasedCall* call)
2064
2017
  : ScopedActivity(call),
2018
+ BatchBuilder(&call->batch_payload_),
2019
+ promise_detail::Context<BatchBuilder>(this),
2065
2020
  promise_detail::Context<Arena>(call->arena()),
2066
2021
  promise_detail::Context<grpc_call_context_element>(call->context_),
2067
2022
  promise_detail::Context<CallContext>(&call->call_context_),
@@ -2095,8 +2050,12 @@ class PromiseBasedCall : public Call,
2095
2050
  };
2096
2051
 
2097
2052
  ~PromiseBasedCall() override {
2098
- if (non_owning_wakeable_) non_owning_wakeable_->DropActivity();
2099
2053
  if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
2054
+ for (int i = 0; i < GRPC_CONTEXT_COUNT; i++) {
2055
+ if (context_[i].destroy) {
2056
+ context_[i].destroy(context_[i].value);
2057
+ }
2058
+ }
2100
2059
  }
2101
2060
 
2102
2061
  // Enumerates why a Completion is still pending
@@ -2104,6 +2063,7 @@ class PromiseBasedCall : public Call,
2104
2063
  // We're in the midst of starting a batch of operations
2105
2064
  kStartingBatch = 0,
2106
2065
  // The following correspond with the batch operations from above
2066
+ kSendInitialMetadata,
2107
2067
  kReceiveInitialMetadata,
2108
2068
  kReceiveStatusOnClient,
2109
2069
  kReceiveCloseOnServer = kReceiveStatusOnClient,
@@ -2113,10 +2073,17 @@ class PromiseBasedCall : public Call,
2113
2073
  kSendCloseFromClient = kSendStatusFromServer,
2114
2074
  };
2115
2075
 
2076
+ bool RunParty() override {
2077
+ ScopedContext ctx(this);
2078
+ return Party::RunParty();
2079
+ }
2080
+
2116
2081
  const char* PendingOpString(PendingOp reason) const {
2117
2082
  switch (reason) {
2118
2083
  case PendingOp::kStartingBatch:
2119
2084
  return "StartingBatch";
2085
+ case PendingOp::kSendInitialMetadata:
2086
+ return "SendInitialMetadata";
2120
2087
  case PendingOp::kReceiveInitialMetadata:
2121
2088
  return "ReceiveInitialMetadata";
2122
2089
  case PendingOp::kReceiveStatusOnClient:
@@ -2131,56 +2098,47 @@ class PromiseBasedCall : public Call,
2131
2098
  return "Unknown";
2132
2099
  }
2133
2100
 
2134
- static constexpr uint8_t PendingOpBit(PendingOp reason) {
2101
+ static constexpr uint32_t PendingOpBit(PendingOp reason) {
2135
2102
  return 1 << static_cast<int>(reason);
2136
2103
  }
2137
2104
 
2138
- Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; }
2139
2105
  // Begin work on a completion, recording the tag/closure to notify.
2140
2106
  // Use the op selected in \a ops to determine the index to allocate into.
2141
2107
  // Starts the "StartingBatch" PendingOp immediately.
2142
2108
  // Assumes at least one operation in \a ops.
2143
- Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops)
2144
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2109
+ Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops);
2145
2110
  // Add one pending op to the completion, and return it.
2146
- Completion AddOpToCompletion(const Completion& completion, PendingOp reason)
2147
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2111
+ Completion AddOpToCompletion(const Completion& completion, PendingOp reason);
2148
2112
  // Stringify a completion
2149
2113
  std::string CompletionString(const Completion& completion) const {
2150
2114
  return completion.has_value()
2151
- ? absl::StrFormat(
2152
- "%d:tag=%p", static_cast<int>(completion.index()),
2153
- completion_info_[completion.index()].pending.tag)
2115
+ ? completion_info_[completion.index()].pending.ToString(this)
2154
2116
  : "no-completion";
2155
2117
  }
2156
2118
  // Finish one op on the completion. Must have been previously been added.
2157
2119
  // The completion as a whole finishes when all pending ops finish.
2158
- void FinishOpOnCompletion(Completion* completion, PendingOp reason)
2159
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2120
+ void FinishOpOnCompletion(Completion* completion, PendingOp reason);
2160
2121
  // Mark the completion as failed. Does not finish it.
2161
2122
  void FailCompletion(const Completion& completion,
2162
2123
  SourceLocation source_location = {});
2163
- // Run the promise polling loop until it stalls.
2164
- void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2165
- // Update the promise state once.
2166
- virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
2124
+ // Mark the completion as infallible. Overrides FailCompletion to report
2125
+ // success always.
2126
+ void ForceCompletionSuccess(const Completion& completion);
2167
2127
  // Accept the stats from the context (call once we have proof the transport is
2168
2128
  // done with them).
2169
2129
  // Right now this means that promise based calls do not record correct stats
2170
2130
  // with census if they are cancelled.
2171
2131
  // TODO(ctiller): this should be remedied before promise based calls are
2172
2132
  // dexperimentalized.
2173
- void AcceptTransportStatsFromContext() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2133
+ void AcceptTransportStatsFromContext() {
2174
2134
  final_stats_ = *call_context_.call_stats();
2175
2135
  }
2176
2136
 
2177
- grpc_completion_queue* cq() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return cq_; }
2137
+ grpc_completion_queue* cq() { return cq_; }
2178
2138
 
2179
2139
  void CToMetadata(grpc_metadata* metadata, size_t count,
2180
2140
  grpc_metadata_batch* batch);
2181
2141
 
2182
- std::string ActivityDebugTag(void*) const override { return DebugTag(); }
2183
-
2184
2142
  // At the end of the call run any finalization actions.
2185
2143
  void RunFinalization(grpc_status_code status, const char* status_details) {
2186
2144
  grpc_call_final_info final_info;
@@ -2209,158 +2167,163 @@ class PromiseBasedCall : public Call,
2209
2167
  }
2210
2168
  }
2211
2169
 
2212
- std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2213
- return absl::StrCat(PresentAndCompletionText("outstanding_send",
2214
- outstanding_send_.has_value(),
2215
- send_message_completion_)
2216
- .c_str(),
2217
- PresentAndCompletionText("outstanding_recv",
2218
- outstanding_recv_.has_value(),
2219
- recv_message_completion_)
2220
- .c_str());
2221
- }
2222
-
2170
+ // Spawn a job that will first do FirstPromise then receive a message
2171
+ template <typename FirstPromise>
2223
2172
  void StartRecvMessage(const grpc_op& op, const Completion& completion,
2224
- PipeReceiver<MessageHandle>* receiver)
2225
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2226
- void PollRecvMessage(grpc_compression_algorithm compression_algorithm)
2227
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2228
- void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2173
+ FirstPromise first,
2174
+ PipeReceiver<MessageHandle>* receiver,
2175
+ Party::BulkSpawner& spawner);
2229
2176
  void StartSendMessage(const grpc_op& op, const Completion& completion,
2230
- PipeSender<MessageHandle>* sender)
2231
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2232
- bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2233
- void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2177
+ PipeSender<MessageHandle>* sender,
2178
+ Party::BulkSpawner& spawner);
2179
+
2180
+ void set_completed() { finished_.Set(); }
2234
2181
 
2235
- bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2236
- return completed_;
2182
+ // Returns a promise that resolves to Empty whenever the call is completed.
2183
+ auto finished() { return finished_.Wait(); }
2184
+
2185
+ // Returns a promise that resolves to Empty whenever there is no outstanding
2186
+ // send operation
2187
+ auto WaitForSendingStarted() {
2188
+ return [this]() -> Poll<Empty> {
2189
+ int n = sends_queued_.load(std::memory_order_relaxed);
2190
+ if (grpc_call_trace.enabled()) {
2191
+ gpr_log(GPR_DEBUG, "%s[call] WaitForSendingStarted n=%d",
2192
+ DebugTag().c_str(), n);
2193
+ }
2194
+ if (n != 0) return waiting_for_queued_sends_.pending();
2195
+ return Empty{};
2196
+ };
2237
2197
  }
2238
- void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; }
2239
- bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2240
- return outstanding_send_.has_value();
2198
+
2199
+ // Mark that a send has been queued - blocks sending trailing metadata.
2200
+ void QueueSend() {
2201
+ if (grpc_call_trace.enabled()) {
2202
+ gpr_log(GPR_DEBUG, "%s[call] QueueSend", DebugTag().c_str());
2203
+ }
2204
+ sends_queued_.fetch_add(1, std::memory_order_relaxed);
2205
+ }
2206
+ // Mark that a send has been dequeued - allows sending trailing metadata once
2207
+ // zero sends are queued.
2208
+ void EnactSend() {
2209
+ if (grpc_call_trace.enabled()) {
2210
+ gpr_log(GPR_DEBUG, "%s[call] EnactSend", DebugTag().c_str());
2211
+ }
2212
+ if (1 == sends_queued_.fetch_sub(1, std::memory_order_relaxed)) {
2213
+ waiting_for_queued_sends_.Wake();
2214
+ }
2241
2215
  }
2242
2216
 
2243
2217
  private:
2244
2218
  union CompletionInfo {
2219
+ static constexpr uint32_t kOpFailed = 0x8000'0000u;
2220
+ static constexpr uint32_t kOpForceSuccess = 0x4000'0000u;
2221
+ CompletionInfo() {}
2222
+ enum CompletionState {
2223
+ kPending,
2224
+ kSuccess,
2225
+ kFailure,
2226
+ };
2245
2227
  struct Pending {
2246
- // Bitmask of PendingOps
2247
- uint8_t pending_op_bits;
2228
+ // Bitmask of PendingOps at the bottom, and kOpFailed, kOpForceSuccess at
2229
+ // the top.
2230
+ std::atomic<uint32_t> state;
2248
2231
  bool is_closure;
2249
- bool success;
2232
+ // True if this completion was for a recv_message op.
2233
+ // In that case if the completion as a whole fails we need to cleanup the
2234
+ // returned message.
2235
+ bool is_recv_message;
2250
2236
  void* tag;
2251
- } pending;
2252
- grpc_cq_completion completion;
2253
- };
2254
2237
 
2255
- class NonOwningWakable final : public Wakeable {
2256
- public:
2257
- explicit NonOwningWakable(PromiseBasedCall* call) : call_(call) {}
2258
-
2259
- // Ref the Handle (not the activity).
2260
- void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
2261
-
2262
- // Activity is going away... drop its reference and sever the connection
2263
- // back.
2264
- void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
2265
- auto unref = absl::MakeCleanup([this]() { Unref(); });
2266
- MutexLock lock(&mu_);
2267
- GPR_ASSERT(call_ != nullptr);
2268
- call_ = nullptr;
2269
- }
2238
+ void Start(bool is_closure, void* tag) {
2239
+ this->is_closure = is_closure;
2240
+ this->is_recv_message = false;
2241
+ this->tag = tag;
2242
+ state.store(PendingOpBit(PendingOp::kStartingBatch),
2243
+ std::memory_order_release);
2244
+ }
2270
2245
 
2271
- // Activity needs to wake up (if it still exists!) - wake it up, and drop
2272
- // the ref that was kept for this handle.
2273
- void Wakeup(void*) override ABSL_LOCKS_EXCLUDED(mu_) {
2274
- // Drop the ref to the handle at end of scope (we have one ref = one
2275
- // wakeup semantics).
2276
- auto unref = absl::MakeCleanup([this]() { Unref(); });
2277
- ReleasableMutexLock lock(&mu_);
2278
- // Note that activity refcount can drop to zero, but we could win the lock
2279
- // against DropActivity, so we need to only increase activities refcount
2280
- // if it is non-zero.
2281
- PromiseBasedCall* call = call_;
2282
- if (call != nullptr && call->RefIfNonZero()) {
2283
- lock.Release();
2284
- // Activity still exists and we have a reference: wake it up, which will
2285
- // drop the ref.
2286
- call->Wakeup(nullptr);
2246
+ void AddPendingBit(PendingOp reason) {
2247
+ if (reason == PendingOp::kReceiveMessage) is_recv_message = true;
2248
+ auto prev =
2249
+ state.fetch_or(PendingOpBit(reason), std::memory_order_relaxed);
2250
+ GPR_ASSERT((prev & PendingOpBit(reason)) == 0);
2287
2251
  }
2288
- }
2289
2252
 
2290
- std::string ActivityDebugTag(void*) const override {
2291
- MutexLock lock(&mu_);
2292
- return call_ == nullptr ? "<unknown>" : call_->DebugTag();
2293
- }
2253
+ CompletionState RemovePendingBit(PendingOp reason) {
2254
+ const uint32_t mask = ~PendingOpBit(reason);
2255
+ auto prev = state.fetch_and(mask, std::memory_order_acq_rel);
2256
+ GPR_ASSERT((prev & PendingOpBit(reason)) != 0);
2257
+ switch (prev & mask) {
2258
+ case kOpFailed:
2259
+ return kFailure;
2260
+ case kOpFailed | kOpForceSuccess:
2261
+ case kOpForceSuccess:
2262
+ case 0:
2263
+ return kSuccess;
2264
+ default:
2265
+ return kPending;
2266
+ }
2267
+ }
2294
2268
 
2295
- void Drop(void*) override { Unref(); }
2269
+ void MarkFailed() {
2270
+ state.fetch_or(kOpFailed, std::memory_order_relaxed);
2271
+ }
2296
2272
 
2297
- private:
2298
- // Unref the Handle (not the activity).
2299
- void Unref() {
2300
- if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) {
2301
- delete this;
2273
+ void MarkForceSuccess() {
2274
+ state.fetch_or(kOpForceSuccess, std::memory_order_relaxed);
2302
2275
  }
2303
- }
2304
2276
 
2305
- mutable Mutex mu_;
2306
- // We have two initial refs: one for the wakeup that this is created for,
2307
- // and will be dropped by Wakeup, and the other for the activity which is
2308
- // dropped by DropActivity.
2309
- std::atomic<size_t> refs_{2};
2310
- PromiseBasedCall* call_ ABSL_GUARDED_BY(mu_);
2277
+ std::string ToString(const PromiseBasedCall* call) const {
2278
+ auto state = this->state.load(std::memory_order_relaxed);
2279
+ std::vector<absl::string_view> pending_ops;
2280
+ for (size_t i = 0; i < 24; i++) {
2281
+ if (state & (1u << i)) {
2282
+ pending_ops.push_back(
2283
+ call->PendingOpString(static_cast<PendingOp>(i)));
2284
+ }
2285
+ }
2286
+ return absl::StrFormat("{%s}%s:tag=%p", absl::StrJoin(pending_ops, ","),
2287
+ (state & kOpForceSuccess) ? ":force-success"
2288
+ : (state & kOpFailed) ? ":failed"
2289
+ : ":success",
2290
+ tag);
2291
+ }
2292
+ } pending;
2293
+ grpc_cq_completion completion;
2311
2294
  };
2312
2295
 
2313
- static void OnDestroy(void* arg, grpc_error_handle) {
2314
- auto* call = static_cast<PromiseBasedCall*>(arg);
2315
- ScopedContext context(call);
2316
- call->DeleteThis();
2317
- }
2318
-
2319
- // First 32 bits are strong refs, next 32 bits are weak refs.
2320
- static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
2321
- return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
2322
- }
2323
- static uint32_t GetStrongRefs(uint64_t ref_pair) {
2324
- return static_cast<uint32_t>(ref_pair >> 32);
2325
- }
2326
- static uint32_t GetWeakRefs(uint64_t ref_pair) {
2327
- return static_cast<uint32_t>(ref_pair & 0xffffffffu);
2328
- }
2329
-
2330
- bool RefIfNonZero() {
2331
- uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire);
2332
- do {
2333
- const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
2334
- if (strong_refs == 0) return false;
2335
- } while (!refs_.compare_exchange_weak(
2336
- prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0),
2337
- std::memory_order_acq_rel, std::memory_order_acquire));
2338
- return true;
2296
+ void PartyOver() override {
2297
+ {
2298
+ ScopedContext ctx(this);
2299
+ CancelRemainingParticipants();
2300
+ arena()->DestroyManagedNewObjects();
2301
+ }
2302
+ DeleteThis();
2339
2303
  }
2340
2304
 
2341
- mutable Mutex mu_;
2342
- std::atomic<uint64_t> refs_;
2343
2305
  CallContext call_context_{this};
2344
- bool keep_polling_ ABSL_GUARDED_BY(mu()) = false;
2345
2306
 
2346
2307
  // Contexts for various subsystems (security, tracing, ...).
2347
2308
  grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
2348
- grpc_completion_queue* cq_ ABSL_GUARDED_BY(mu_);
2349
- NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr;
2309
+ grpc_completion_queue* cq_;
2350
2310
  CompletionInfo completion_info_[6];
2351
2311
  grpc_call_stats final_stats_{};
2352
2312
  CallFinalization finalization_;
2353
2313
  // Current deadline.
2354
- Timestamp deadline_ = Timestamp::InfFuture();
2355
- grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_;
2356
- absl::optional<PipeSender<MessageHandle>::PushType> outstanding_send_
2357
- ABSL_GUARDED_BY(mu_);
2358
- absl::optional<PipeReceiverNextType<MessageHandle>> outstanding_recv_
2359
- ABSL_GUARDED_BY(mu_);
2360
- grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr;
2361
- Completion send_message_completion_ ABSL_GUARDED_BY(mu_);
2362
- Completion recv_message_completion_ ABSL_GUARDED_BY(mu_);
2363
- bool completed_ ABSL_GUARDED_BY(mu_) = false;
2314
+ Mutex deadline_mu_;
2315
+ Timestamp deadline_ ABSL_GUARDED_BY(deadline_mu_) = Timestamp::InfFuture();
2316
+ grpc_event_engine::experimental::EventEngine::TaskHandle ABSL_GUARDED_BY(
2317
+ deadline_mu_) deadline_task_;
2318
+ ExternallyObservableLatch<void> finished_;
2319
+ // Non-zero with an outstanding GRPC_OP_SEND_INITIAL_METADATA or
2320
+ // GRPC_OP_SEND_MESSAGE (one count each), and 0 once those payloads have been
2321
+ // pushed onto the outgoing pipe.
2322
+ std::atomic<uint8_t> sends_queued_{0};
2323
+ // Waiter for when sends_queued_ becomes 0.
2324
+ IntraActivityWaiter waiting_for_queued_sends_;
2325
+ grpc_byte_buffer** recv_message_ = nullptr;
2326
+ grpc_transport_stream_op_batch_payload batch_payload_{context_};
2364
2327
  };
2365
2328
 
2366
2329
  template <typename T>
@@ -2380,7 +2343,7 @@ PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
2380
2343
  const grpc_call_create_args& args)
2381
2344
  : Call(arena, args.server_transport_data == nullptr, args.send_deadline,
2382
2345
  args.channel->Ref()),
2383
- refs_(MakeRefPair(initial_external_refs, 0)),
2346
+ Party(arena, initial_external_refs),
2384
2347
  cq_(args.cq) {
2385
2348
  if (args.cq != nullptr) {
2386
2349
  GPR_ASSERT(args.pollset_set_alternative == nullptr &&
@@ -2396,15 +2359,6 @@ PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
2396
2359
  }
2397
2360
  }
2398
2361
 
2399
- Waker PromiseBasedCall::MakeNonOwningWaker() {
2400
- if (non_owning_wakeable_ == nullptr) {
2401
- non_owning_wakeable_ = new NonOwningWakable(this);
2402
- } else {
2403
- non_owning_wakeable_->Ref();
2404
- }
2405
- return Waker(non_owning_wakeable_, nullptr);
2406
- }
2407
-
2408
2362
  void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count,
2409
2363
  grpc_metadata_batch* b) {
2410
2364
  for (size_t i = 0; i < count; i++) {
@@ -2439,15 +2393,14 @@ void* PromiseBasedCall::ContextGet(grpc_context_index elem) const {
2439
2393
  PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
2440
2394
  void* tag, bool is_closure, const grpc_op* ops) {
2441
2395
  Completion c(BatchSlotForOp(ops[0].op));
2442
- if (grpc_call_trace.enabled()) {
2443
- gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(),
2444
- CompletionString(c).c_str(), tag);
2445
- }
2446
2396
  if (!is_closure) {
2447
2397
  grpc_cq_begin_op(cq(), tag);
2448
2398
  }
2449
- completion_info_[c.index()].pending = {
2450
- PendingOpBit(PendingOp::kStartingBatch), is_closure, true, tag};
2399
+ completion_info_[c.index()].pending.Start(is_closure, tag);
2400
+ if (grpc_call_trace.enabled()) {
2401
+ gpr_log(GPR_INFO, "%s[call] StartCompletion %s", DebugTag().c_str(),
2402
+ CompletionString(c).c_str());
2403
+ }
2451
2404
  return c;
2452
2405
  }
2453
2406
 
@@ -2458,10 +2411,7 @@ PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
2458
2411
  CompletionString(completion).c_str(), PendingOpString(reason));
2459
2412
  }
2460
2413
  GPR_ASSERT(completion.has_value());
2461
- auto& pending_op_bits =
2462
- completion_info_[completion.index()].pending.pending_op_bits;
2463
- GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0);
2464
- pending_op_bits |= PendingOpBit(reason);
2414
+ completion_info_[completion.index()].pending.AddPendingBit(reason);
2465
2415
  return Completion(completion.index());
2466
2416
  }
2467
2417
 
@@ -2472,64 +2422,50 @@ void PromiseBasedCall::FailCompletion(const Completion& completion,
2472
2422
  "%s[call] FailCompletion %s", DebugTag().c_str(),
2473
2423
  CompletionString(completion).c_str());
2474
2424
  }
2475
- completion_info_[completion.index()].pending.success = false;
2425
+ completion_info_[completion.index()].pending.MarkFailed();
2426
+ }
2427
+
2428
+ void PromiseBasedCall::ForceCompletionSuccess(const Completion& completion) {
2429
+ completion_info_[completion.index()].pending.MarkForceSuccess();
2476
2430
  }
2477
2431
 
2478
2432
  void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
2479
2433
  PendingOp reason) {
2480
2434
  if (grpc_call_trace.enabled()) {
2481
- auto pending_op_bits =
2482
- completion_info_[completion->index()].pending.pending_op_bits;
2483
- bool success = completion_info_[completion->index()].pending.success;
2484
- std::vector<const char*> pending;
2485
- for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) {
2486
- if (static_cast<PendingOp>(i) == reason) continue;
2487
- if (pending_op_bits & (1 << i)) {
2488
- pending.push_back(PendingOpString(static_cast<PendingOp>(i)));
2489
- }
2490
- }
2491
- gpr_log(
2492
- GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s",
2493
- DebugTag().c_str(), completion_info_[completion->index()].pending.tag,
2494
- CompletionString(*completion).c_str(), PendingOpString(reason),
2495
- (pending.empty()
2496
- ? (success ? std::string("done") : std::string("failed"))
2497
- : absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ",")))
2498
- .c_str());
2435
+ gpr_log(GPR_INFO, "%s[call] FinishOpOnCompletion completion:%s finish:%s",
2436
+ DebugTag().c_str(), CompletionString(*completion).c_str(),
2437
+ PendingOpString(reason));
2499
2438
  }
2500
2439
  const uint8_t i = completion->TakeIndex();
2501
2440
  GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_));
2502
2441
  CompletionInfo::Pending& pending = completion_info_[i].pending;
2503
- GPR_ASSERT(pending.pending_op_bits & PendingOpBit(reason));
2504
- pending.pending_op_bits &= ~PendingOpBit(reason);
2505
- auto error = pending.success ? absl::OkStatus() : absl::CancelledError();
2506
- if (pending.pending_op_bits == 0) {
2507
- if (pending.is_closure) {
2508
- ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
2509
- error);
2510
- } else {
2511
- grpc_cq_end_op(
2512
- cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
2513
- &completion_info_[i].completion);
2514
- }
2442
+ bool success;
2443
+ switch (pending.RemovePendingBit(reason)) {
2444
+ case CompletionInfo::kPending:
2445
+ return; // Early out
2446
+ case CompletionInfo::kSuccess:
2447
+ success = true;
2448
+ break;
2449
+ case CompletionInfo::kFailure:
2450
+ success = false;
2451
+ break;
2452
+ }
2453
+ if (pending.is_recv_message && !success && *recv_message_ != nullptr) {
2454
+ grpc_byte_buffer_destroy(*recv_message_);
2455
+ *recv_message_ = nullptr;
2456
+ }
2457
+ auto error = success ? absl::OkStatus() : absl::CancelledError();
2458
+ if (pending.is_closure) {
2459
+ ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
2460
+ error);
2461
+ } else {
2462
+ grpc_cq_end_op(
2463
+ cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
2464
+ &completion_info_[i].completion);
2515
2465
  }
2516
2466
  }
2517
2467
 
2518
- void PromiseBasedCall::Update() {
2519
- keep_polling_ = false;
2520
- do {
2521
- UpdateOnce();
2522
- } while (std::exchange(keep_polling_, false));
2523
- }
2524
-
2525
- void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; }
2526
-
2527
2468
  void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) {
2528
- MutexLock lock(&mu_);
2529
- SetCompletionQueueLocked(cq);
2530
- }
2531
-
2532
- void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
2533
2469
  cq_ = cq;
2534
2470
  GRPC_CQ_INTERNAL_REF(cq, "bind");
2535
2471
  call_context_.pollent_ =
@@ -2537,6 +2473,12 @@ void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
2537
2473
  }
2538
2474
 
2539
2475
  void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
2476
+ MutexLock lock(&deadline_mu_);
2477
+ if (grpc_call_trace.enabled()) {
2478
+ gpr_log(GPR_DEBUG, "%s[call] UpdateDeadline from=%s to=%s",
2479
+ DebugTag().c_str(), deadline_.ToString().c_str(),
2480
+ deadline.ToString().c_str());
2481
+ }
2540
2482
  if (deadline >= deadline_) return;
2541
2483
  auto* const event_engine = channel()->event_engine();
2542
2484
  if (deadline_ != Timestamp::InfFuture()) {
@@ -2544,10 +2486,12 @@ void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
2544
2486
  } else {
2545
2487
  InternalRef("deadline");
2546
2488
  }
2547
- event_engine->RunAfter(deadline - Timestamp::Now(), this);
2489
+ deadline_ = deadline;
2490
+ deadline_task_ = event_engine->RunAfter(deadline - Timestamp::Now(), this);
2548
2491
  }
2549
2492
 
2550
2493
  void PromiseBasedCall::ResetDeadline() {
2494
+ MutexLock lock(&deadline_mu_);
2551
2495
  if (deadline_ == Timestamp::InfFuture()) return;
2552
2496
  auto* const event_engine = channel()->event_engine();
2553
2497
  if (!event_engine->Cancel(deadline_task_)) return;
@@ -2564,117 +2508,88 @@ void PromiseBasedCall::Run() {
2564
2508
 
2565
2509
  void PromiseBasedCall::StartSendMessage(const grpc_op& op,
2566
2510
  const Completion& completion,
2567
- PipeSender<MessageHandle>* sender) {
2568
- GPR_ASSERT(!outstanding_send_.has_value());
2569
- if (!completed_) {
2570
- send_message_completion_ =
2571
- AddOpToCompletion(completion, PendingOp::kSendMessage);
2572
- SliceBuffer send;
2573
- grpc_slice_buffer_swap(
2574
- &op.data.send_message.send_message->data.raw.slice_buffer,
2575
- send.c_slice_buffer());
2576
- outstanding_send_.emplace(sender->Push(
2577
- GetContext<Arena>()->MakePooled<Message>(std::move(send), op.flags)));
2578
- } else {
2579
- FailCompletion(completion);
2580
- }
2511
+ PipeSender<MessageHandle>* sender,
2512
+ Party::BulkSpawner& spawner) {
2513
+ QueueSend();
2514
+ SliceBuffer send;
2515
+ grpc_slice_buffer_swap(
2516
+ &op.data.send_message.send_message->data.raw.slice_buffer,
2517
+ send.c_slice_buffer());
2518
+ auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
2519
+ spawner.Spawn(
2520
+ "call_send_message",
2521
+ [this, sender, msg = std::move(msg)]() mutable {
2522
+ EnactSend();
2523
+ return sender->Push(std::move(msg));
2524
+ },
2525
+ [this, completion = AddOpToCompletion(
2526
+ completion, PendingOp::kSendMessage)](bool result) mutable {
2527
+ if (grpc_call_trace.enabled()) {
2528
+ gpr_log(GPR_DEBUG, "%sSendMessage completes %s", DebugTag().c_str(),
2529
+ result ? "successfully" : "with failure");
2530
+ }
2531
+ if (!result) FailCompletion(completion);
2532
+ FinishOpOnCompletion(&completion, PendingOp::kSendMessage);
2533
+ });
2581
2534
  }
2582
2535
 
2583
- bool PromiseBasedCall::PollSendMessage() {
2584
- if (!outstanding_send_.has_value()) return true;
2585
- Poll<bool> r = (*outstanding_send_)();
2586
- if (const bool* result = r.value_if_ready()) {
2587
- if (grpc_call_trace.enabled()) {
2588
- gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(),
2589
- *result ? "successfully" : "with failure");
2590
- }
2591
- if (!*result) {
2592
- FailCompletion(send_message_completion_);
2593
- return false;
2594
- }
2595
- FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
2596
- outstanding_send_.reset();
2536
+ template <typename FirstPromiseFactory>
2537
+ void PromiseBasedCall::StartRecvMessage(
2538
+ const grpc_op& op, const Completion& completion,
2539
+ FirstPromiseFactory first_promise_factory,
2540
+ PipeReceiver<MessageHandle>* receiver, Party::BulkSpawner& spawner) {
2541
+ if (grpc_call_trace.enabled()) {
2542
+ gpr_log(GPR_INFO, "%s[call] Start RecvMessage: %s", DebugTag().c_str(),
2543
+ CompletionString(completion).c_str());
2597
2544
  }
2598
- return true;
2599
- }
2600
-
2601
- void PromiseBasedCall::CancelSendMessage() {
2602
- if (!outstanding_send_.has_value()) return;
2603
- FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
2604
- outstanding_send_.reset();
2605
- }
2606
-
2607
- void PromiseBasedCall::StartRecvMessage(const grpc_op& op,
2608
- const Completion& completion,
2609
- PipeReceiver<MessageHandle>* receiver) {
2610
- GPR_ASSERT(!outstanding_recv_.has_value());
2611
2545
  recv_message_ = op.data.recv_message.recv_message;
2612
- recv_message_completion_ =
2613
- AddOpToCompletion(completion, PendingOp::kReceiveMessage);
2614
- outstanding_recv_.emplace(receiver->Next());
2615
- }
2616
-
2617
- void PromiseBasedCall::PollRecvMessage(
2618
- grpc_compression_algorithm incoming_compression_algorithm) {
2619
- if (!outstanding_recv_.has_value()) return;
2620
- Poll<NextResult<MessageHandle>> r = (*outstanding_recv_)();
2621
- if (auto* result = r.value_if_ready()) {
2622
- outstanding_recv_.reset();
2623
- if (result->has_value()) {
2624
- MessageHandle& message = **result;
2625
- if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
2626
- (incoming_compression_algorithm != GRPC_COMPRESS_NONE)) {
2627
- *recv_message_ = grpc_raw_compressed_byte_buffer_create(
2628
- nullptr, 0, incoming_compression_algorithm);
2629
- } else {
2630
- *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
2631
- }
2632
- grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
2633
- &(*recv_message_)->data.raw.slice_buffer);
2634
- if (grpc_call_trace.enabled()) {
2635
- gpr_log(GPR_INFO,
2636
- "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2637
- "%" PRIdPTR " byte message",
2638
- DebugTag().c_str(),
2639
- (*recv_message_)->data.raw.slice_buffer.length);
2640
- }
2641
- } else if (result->cancelled()) {
2642
- if (grpc_call_trace.enabled()) {
2643
- gpr_log(GPR_INFO,
2644
- "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2645
- "end-of-stream with error",
2646
- DebugTag().c_str());
2647
- }
2648
- FailCompletion(recv_message_completion_);
2649
- *recv_message_ = nullptr;
2650
- } else {
2651
- if (grpc_call_trace.enabled()) {
2652
- gpr_log(GPR_INFO,
2653
- "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2654
- "end-of-stream",
2655
- DebugTag().c_str());
2656
- }
2657
- *recv_message_ = nullptr;
2658
- }
2659
- FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2660
- } else if (completed_) {
2661
- if (grpc_call_trace.enabled()) {
2662
- gpr_log(GPR_INFO,
2663
- "%s[call] UpdateOnce: outstanding_recv finishes: promise has "
2664
- "completed without queuing a message, forcing end-of-stream",
2665
- DebugTag().c_str());
2666
- }
2667
- outstanding_recv_.reset();
2668
- *recv_message_ = nullptr;
2669
- FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2670
- }
2671
- }
2672
-
2673
- void PromiseBasedCall::CancelRecvMessage() {
2674
- if (!outstanding_recv_.has_value()) return;
2675
- *recv_message_ = nullptr;
2676
- outstanding_recv_.reset();
2677
- FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2546
+ spawner.Spawn(
2547
+ "call_recv_message",
2548
+ [first_promise_factory = std::move(first_promise_factory), receiver]() {
2549
+ return Seq(first_promise_factory(), receiver->Next());
2550
+ },
2551
+ [this,
2552
+ completion = AddOpToCompletion(completion, PendingOp::kReceiveMessage)](
2553
+ NextResult<MessageHandle> result) mutable {
2554
+ if (result.has_value()) {
2555
+ MessageHandle& message = *result;
2556
+ NoteLastMessageFlags(message->flags());
2557
+ if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
2558
+ (incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
2559
+ *recv_message_ = grpc_raw_compressed_byte_buffer_create(
2560
+ nullptr, 0, incoming_compression_algorithm());
2561
+ } else {
2562
+ *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
2563
+ }
2564
+ grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
2565
+ &(*recv_message_)->data.raw.slice_buffer);
2566
+ if (grpc_call_trace.enabled()) {
2567
+ gpr_log(GPR_INFO,
2568
+ "%s[call] RecvMessage: outstanding_recv "
2569
+ "finishes: received %" PRIdPTR " byte message",
2570
+ DebugTag().c_str(),
2571
+ (*recv_message_)->data.raw.slice_buffer.length);
2572
+ }
2573
+ } else if (result.cancelled()) {
2574
+ if (grpc_call_trace.enabled()) {
2575
+ gpr_log(GPR_INFO,
2576
+ "%s[call] RecvMessage: outstanding_recv "
2577
+ "finishes: received end-of-stream with error",
2578
+ DebugTag().c_str());
2579
+ }
2580
+ FailCompletion(completion);
2581
+ *recv_message_ = nullptr;
2582
+ } else {
2583
+ if (grpc_call_trace.enabled()) {
2584
+ gpr_log(GPR_INFO,
2585
+ "%s[call] RecvMessage: outstanding_recv "
2586
+ "finishes: received end-of-stream",
2587
+ DebugTag().c_str());
2588
+ }
2589
+ *recv_message_ = nullptr;
2590
+ }
2591
+ FinishOpOnCompletion(&completion, PendingOp::kReceiveMessage);
2592
+ });
2678
2593
  }
2679
2594
 
2680
2595
  ///////////////////////////////////////////////////////////////////////////////
@@ -2690,10 +2605,6 @@ void CallContext::IncrementRefCount(const char* reason) {
2690
2605
 
2691
2606
  void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
2692
2607
 
2693
- gpr_atm* CallContext::peer_string_atm_ptr() {
2694
- return call_->peer_string_atm_ptr();
2695
- }
2696
-
2697
2608
  void CallContext::UpdateDeadline(Timestamp deadline) {
2698
2609
  call_->UpdateDeadline(deadline);
2699
2610
  }
@@ -2747,24 +2658,40 @@ class ClientPromiseBasedCall final : public PromiseBasedCall {
2747
2658
  ~ClientPromiseBasedCall() override {
2748
2659
  ScopedContext context(this);
2749
2660
  send_initial_metadata_.reset();
2750
- recv_status_on_client_ = absl::monostate();
2751
- promise_ = ArenaPromise<ServerMetadataHandle>();
2752
- // Need to destroy the pipes under the ScopedContext above, so we move them
2753
- // out here and then allow the destructors to run at end of scope, but
2754
- // before context.
2661
+ // Need to destroy the pipes under the ScopedContext above, so we
2662
+ // move them out here and then allow the destructors to run at
2663
+ // end of scope, but before context.
2755
2664
  auto c2s = std::move(client_to_server_messages_);
2756
2665
  auto s2c = std::move(server_to_client_messages_);
2757
2666
  auto sim = std::move(server_initial_metadata_);
2758
2667
  }
2759
2668
 
2760
- absl::string_view GetServerAuthority() const override { abort(); }
2761
- void CancelWithErrorLocked(grpc_error_handle error) override
2762
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2763
- bool is_trailers_only() const override {
2764
- MutexLock lock(mu());
2765
- return is_trailers_only_;
2669
+ void CancelWithError(absl::Status error) override {
2670
+ if (!started_.exchange(true, std::memory_order_relaxed)) {
2671
+ // Initial metadata not sent yet, so we can just fail the call.
2672
+ Spawn(
2673
+ "cancel_before_initial_metadata",
2674
+ [error = std::move(error), this]() {
2675
+ server_to_client_messages_.sender.Close();
2676
+ Finish(ServerMetadataFromStatus(error));
2677
+ return Empty{};
2678
+ },
2679
+ [](Empty) {});
2680
+ } else {
2681
+ Spawn(
2682
+ "cancel_with_error",
2683
+ [error = std::move(error), this]() {
2684
+ if (!cancel_error_.is_set()) {
2685
+ cancel_error_.Set(ServerMetadataFromStatus(error));
2686
+ }
2687
+ return Empty{};
2688
+ },
2689
+ [](Empty) {});
2690
+ }
2766
2691
  }
2767
- bool failed_before_recv_message() const override { abort(); }
2692
+ absl::string_view GetServerAuthority() const override { abort(); }
2693
+ bool is_trailers_only() const override { return is_trailers_only_; }
2694
+ bool failed_before_recv_message() const override { return false; }
2768
2695
 
2769
2696
  grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
2770
2697
  bool is_notify_tag_closure) override;
@@ -2774,65 +2701,79 @@ class ClientPromiseBasedCall final : public PromiseBasedCall {
2774
2701
  }
2775
2702
 
2776
2703
  private:
2777
- // Poll the underlying promise (and sundry objects) once.
2778
- void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
2779
2704
  // Finish the call with the given status/trailing metadata.
2780
- void Finish(ServerMetadataHandle trailing_metadata)
2781
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2705
+ void Finish(ServerMetadataHandle trailing_metadata);
2782
2706
  // Validate that a set of ops is valid for a client call.
2783
- grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const
2784
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2707
+ grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
2785
2708
  // Commit a valid batch of operations to be executed.
2786
2709
  void CommitBatch(const grpc_op* ops, size_t nops,
2787
- const Completion& completion)
2788
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2710
+ const Completion& completion);
2789
2711
  // Start the underlying promise.
2790
- void StartPromise(ClientMetadataHandle client_initial_metadata)
2791
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2712
+ void StartPromise(ClientMetadataHandle client_initial_metadata,
2713
+ const Completion& completion, Party::BulkSpawner& spawner);
2714
+ // Start receiving initial metadata
2715
+ void StartRecvInitialMetadata(grpc_metadata_array* array,
2716
+ const Completion& completion,
2717
+ Party::BulkSpawner& spawner);
2718
+ void StartRecvStatusOnClient(
2719
+ const Completion& completion,
2720
+ grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
2721
+ Party::BulkSpawner& spawner);
2792
2722
  // Publish status out to the application.
2793
2723
  void PublishStatus(
2794
2724
  grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
2795
- ServerMetadataHandle trailing_metadata)
2796
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2725
+ ServerMetadataHandle trailing_metadata);
2797
2726
  // Publish server initial metadata out to the application.
2798
- void PublishInitialMetadata(ServerMetadata* metadata)
2799
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2800
-
2801
- ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
2802
- Pipe<ServerMetadataHandle> server_initial_metadata_ ABSL_GUARDED_BY(mu()){
2803
- arena()};
2804
- Pipe<MessageHandle> client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()};
2805
- Pipe<MessageHandle> server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()};
2727
+ void PublishInitialMetadata(ServerMetadata* metadata);
2806
2728
 
2807
2729
  ClientMetadataHandle send_initial_metadata_;
2808
- grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr;
2809
- absl::variant<absl::monostate,
2810
- grpc_op::grpc_op_data::grpc_op_recv_status_on_client,
2811
- ServerMetadataHandle>
2812
- recv_status_on_client_ ABSL_GUARDED_BY(mu());
2813
- absl::optional<PipeReceiverNextType<ServerMetadataHandle>>
2814
- server_initial_metadata_ready_;
2815
- absl::optional<grpc_compression_algorithm> incoming_compression_algorithm_;
2816
- Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu());
2817
- Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu());
2818
- Completion close_send_completion_ ABSL_GUARDED_BY(mu());
2819
- bool is_trailers_only_ ABSL_GUARDED_BY(mu());
2730
+ Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
2731
+ Latch<ServerMetadataHandle> server_trailing_metadata_;
2732
+ Latch<ServerMetadataHandle> cancel_error_;
2733
+ Pipe<MessageHandle> client_to_server_messages_{arena()};
2734
+ Pipe<MessageHandle> server_to_client_messages_{arena()};
2735
+ bool is_trailers_only_;
2736
+ // True once the promise for the call is started.
2737
+ // This corresponds to sending initial metadata, or cancelling before doing
2738
+ // so.
2739
+ // In the latter case real world code sometimes does not sent the initial
2740
+ // metadata, and so gating based upon that does not work out.
2741
+ std::atomic<bool> started_{false};
2820
2742
  };
2821
2743
 
2822
2744
  void ClientPromiseBasedCall::StartPromise(
2823
- ClientMetadataHandle client_initial_metadata) {
2824
- GPR_ASSERT(!promise_.has_value());
2825
- promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{
2826
- std::move(client_initial_metadata),
2827
- &server_initial_metadata_.sender,
2828
- &client_to_server_messages_.receiver,
2829
- &server_to_client_messages_.sender,
2830
- });
2831
- }
2832
-
2833
- void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) {
2834
- ScopedContext context(this);
2835
- Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error)));
2745
+ ClientMetadataHandle client_initial_metadata, const Completion& completion,
2746
+ Party::BulkSpawner& spawner) {
2747
+ auto token = ClientInitialMetadataOutstandingToken::New(arena());
2748
+ spawner.Spawn(
2749
+ "call_send_initial_metadata", token.Wait(),
2750
+ [this,
2751
+ completion = AddOpToCompletion(
2752
+ completion, PendingOp::kSendInitialMetadata)](bool result) mutable {
2753
+ if (!result) FailCompletion(completion);
2754
+ FinishOpOnCompletion(&completion, PendingOp::kSendInitialMetadata);
2755
+ });
2756
+ spawner.Spawn(
2757
+ "client_promise",
2758
+ [this, client_initial_metadata = std::move(client_initial_metadata),
2759
+ token = std::move(token)]() mutable {
2760
+ return Race(
2761
+ cancel_error_.Wait(),
2762
+ Map(channel()->channel_stack()->MakeClientCallPromise(
2763
+ CallArgs{std::move(client_initial_metadata),
2764
+ std::move(token), &server_initial_metadata_.sender,
2765
+ &client_to_server_messages_.receiver,
2766
+ &server_to_client_messages_.sender}),
2767
+ [this](ServerMetadataHandle trailing_metadata) {
2768
+ // If we're cancelled the transport doesn't get to return
2769
+ // stats.
2770
+ AcceptTransportStatsFromContext();
2771
+ return trailing_metadata;
2772
+ }));
2773
+ },
2774
+ [this](ServerMetadataHandle trailing_metadata) {
2775
+ Finish(std::move(trailing_metadata));
2776
+ });
2836
2777
  }
2837
2778
 
2838
2779
  grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
@@ -2873,49 +2814,61 @@ grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
2873
2814
 
2874
2815
  void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
2875
2816
  const Completion& completion) {
2817
+ Party::BulkSpawner spawner(this);
2876
2818
  for (size_t op_idx = 0; op_idx < nops; op_idx++) {
2877
2819
  const grpc_op& op = ops[op_idx];
2878
2820
  switch (op.op) {
2879
2821
  case GRPC_OP_SEND_INITIAL_METADATA: {
2880
- // compression not implemented
2881
- GPR_ASSERT(
2882
- !op.data.send_initial_metadata.maybe_compression_level.is_set);
2883
- if (!completed()) {
2884
- CToMetadata(op.data.send_initial_metadata.metadata,
2885
- op.data.send_initial_metadata.count,
2886
- send_initial_metadata_.get());
2887
- StartPromise(std::move(send_initial_metadata_));
2822
+ if (started_.exchange(true, std::memory_order_relaxed)) break;
2823
+ CToMetadata(op.data.send_initial_metadata.metadata,
2824
+ op.data.send_initial_metadata.count,
2825
+ send_initial_metadata_.get());
2826
+ PrepareOutgoingInitialMetadata(op, *send_initial_metadata_);
2827
+ if (send_deadline() != Timestamp::InfFuture()) {
2828
+ send_initial_metadata_->Set(GrpcTimeoutMetadata(), send_deadline());
2888
2829
  }
2830
+ send_initial_metadata_->Set(
2831
+ WaitForReady(),
2832
+ WaitForReady::ValueType{
2833
+ (op.flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
2834
+ (op.flags &
2835
+ GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
2836
+ StartPromise(std::move(send_initial_metadata_), completion, spawner);
2889
2837
  } break;
2890
2838
  case GRPC_OP_RECV_INITIAL_METADATA: {
2891
- recv_initial_metadata_ =
2892
- op.data.recv_initial_metadata.recv_initial_metadata;
2893
- server_initial_metadata_ready_.emplace(
2894
- server_initial_metadata_.receiver.Next());
2895
- recv_initial_metadata_completion_ =
2896
- AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata);
2839
+ StartRecvInitialMetadata(
2840
+ op.data.recv_initial_metadata.recv_initial_metadata, completion,
2841
+ spawner);
2897
2842
  } break;
2898
2843
  case GRPC_OP_RECV_STATUS_ON_CLIENT: {
2899
- recv_status_on_client_completion_ =
2900
- AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient);
2901
- if (auto* finished_metadata =
2902
- absl::get_if<ServerMetadataHandle>(&recv_status_on_client_)) {
2903
- PublishStatus(op.data.recv_status_on_client,
2904
- std::move(*finished_metadata));
2905
- } else {
2906
- recv_status_on_client_ = op.data.recv_status_on_client;
2907
- }
2844
+ StartRecvStatusOnClient(completion, op.data.recv_status_on_client,
2845
+ spawner);
2908
2846
  } break;
2909
2847
  case GRPC_OP_SEND_MESSAGE:
2910
- StartSendMessage(op, completion, &client_to_server_messages_.sender);
2848
+ StartSendMessage(op, completion, &client_to_server_messages_.sender,
2849
+ spawner);
2911
2850
  break;
2912
2851
  case GRPC_OP_RECV_MESSAGE:
2913
- StartRecvMessage(op, completion, &server_to_client_messages_.receiver);
2852
+ StartRecvMessage(
2853
+ op, completion,
2854
+ [this]() {
2855
+ return server_initial_metadata_.receiver.AwaitClosed();
2856
+ },
2857
+ &server_to_client_messages_.receiver, spawner);
2914
2858
  break;
2915
2859
  case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
2916
- close_send_completion_ =
2917
- AddOpToCompletion(completion, PendingOp::kSendCloseFromClient);
2918
- GPR_ASSERT(close_send_completion_.has_value());
2860
+ spawner.Spawn(
2861
+ "send_close_from_client",
2862
+ [this]() {
2863
+ client_to_server_messages_.sender.Close();
2864
+ return Empty{};
2865
+ },
2866
+ [this,
2867
+ completion = AddOpToCompletion(
2868
+ completion, PendingOp::kSendCloseFromClient)](Empty) mutable {
2869
+ FinishOpOnCompletion(&completion,
2870
+ PendingOp::kSendCloseFromClient);
2871
+ });
2919
2872
  break;
2920
2873
  case GRPC_OP_SEND_STATUS_FROM_SERVER:
2921
2874
  case GRPC_OP_RECV_CLOSE_ON_SERVER:
@@ -2928,8 +2881,6 @@ grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
2928
2881
  size_t nops,
2929
2882
  void* notify_tag,
2930
2883
  bool is_notify_tag_closure) {
2931
- MutexLock lock(mu());
2932
- ScopedContext activity_context(this);
2933
2884
  if (nops == 0) {
2934
2885
  EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
2935
2886
  return GRPC_CALL_OK;
@@ -2941,69 +2892,35 @@ grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
2941
2892
  Completion completion =
2942
2893
  StartCompletion(notify_tag, is_notify_tag_closure, ops);
2943
2894
  CommitBatch(ops, nops, completion);
2944
- Update();
2945
2895
  FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
2946
2896
  return GRPC_CALL_OK;
2947
2897
  }
2948
2898
 
2949
- void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) {
2950
- incoming_compression_algorithm_ =
2951
- metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
2952
- server_initial_metadata_ready_.reset();
2953
- GPR_ASSERT(recv_initial_metadata_ != nullptr);
2954
- PublishMetadataArray(metadata,
2955
- std::exchange(recv_initial_metadata_, nullptr));
2956
- FinishOpOnCompletion(&recv_initial_metadata_completion_,
2957
- PendingOp::kReceiveInitialMetadata);
2958
- }
2959
-
2960
- void ClientPromiseBasedCall::UpdateOnce() {
2961
- if (grpc_call_trace.enabled()) {
2962
- gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s",
2963
- DebugTag().c_str(),
2964
- PresentAndCompletionText("server_initial_metadata_ready",
2965
- server_initial_metadata_ready_.has_value(),
2966
- recv_initial_metadata_completion_)
2967
- .c_str(),
2968
- PollStateDebugString().c_str(),
2969
- promise_.has_value() ? "true" : "false");
2970
- }
2971
- if (server_initial_metadata_ready_.has_value()) {
2972
- Poll<NextResult<ServerMetadataHandle>> r =
2973
- (*server_initial_metadata_ready_)();
2974
- if (auto* server_initial_metadata = r.value_if_ready()) {
2975
- PublishInitialMetadata(server_initial_metadata->value().get());
2976
- } else if (completed()) {
2977
- ServerMetadata no_metadata{GetContext<Arena>()};
2978
- PublishInitialMetadata(&no_metadata);
2979
- }
2980
- }
2981
- if (!PollSendMessage()) {
2982
- Finish(ServerMetadataFromStatus(absl::Status(
2983
- absl::StatusCode::kInternal, "Failed to send message to server")));
2984
- }
2985
- if (!is_sending() && close_send_completion_.has_value()) {
2986
- client_to_server_messages_.sender.Close();
2987
- FinishOpOnCompletion(&close_send_completion_,
2988
- PendingOp::kSendCloseFromClient);
2989
- }
2990
- if (promise_.has_value()) {
2991
- Poll<ServerMetadataHandle> r = promise_();
2992
- if (grpc_call_trace.enabled()) {
2993
- gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
2994
- DebugTag().c_str(),
2995
- PollToString(r, [](const ServerMetadataHandle& h) {
2996
- return h->DebugString();
2997
- }).c_str());
2998
- }
2999
- if (auto* result = r.value_if_ready()) {
3000
- AcceptTransportStatsFromContext();
3001
- Finish(std::move(*result));
3002
- }
3003
- }
3004
- if (incoming_compression_algorithm_.has_value()) {
3005
- PollRecvMessage(*incoming_compression_algorithm_);
3006
- }
2899
+ void ClientPromiseBasedCall::StartRecvInitialMetadata(
2900
+ grpc_metadata_array* array, const Completion& completion,
2901
+ Party::BulkSpawner& spawner) {
2902
+ spawner.Spawn(
2903
+ "recv_initial_metadata",
2904
+ Race(server_initial_metadata_.receiver.Next(),
2905
+ Map(finished(),
2906
+ [](Empty) { return NextResult<ServerMetadataHandle>(true); })),
2907
+ [this, array,
2908
+ completion =
2909
+ AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata)](
2910
+ NextResult<ServerMetadataHandle> next_metadata) mutable {
2911
+ server_initial_metadata_.sender.Close();
2912
+ ServerMetadataHandle metadata;
2913
+ if (next_metadata.has_value()) {
2914
+ is_trailers_only_ = false;
2915
+ metadata = std::move(next_metadata.value());
2916
+ } else {
2917
+ is_trailers_only_ = true;
2918
+ metadata = arena()->MakePooled<ServerMetadata>(arena());
2919
+ }
2920
+ ProcessIncomingInitialMetadata(*metadata);
2921
+ PublishMetadataArray(metadata.get(), array);
2922
+ FinishOpOnCompletion(&completion, PendingOp::kReceiveInitialMetadata);
2923
+ });
3007
2924
  }
3008
2925
 
3009
2926
  void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
@@ -3011,31 +2928,9 @@ void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
3011
2928
  gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
3012
2929
  trailing_metadata->DebugString().c_str());
3013
2930
  }
3014
- promise_ = ArenaPromise<ServerMetadataHandle>();
3015
2931
  ResetDeadline();
3016
2932
  set_completed();
3017
- if (recv_initial_metadata_ != nullptr) {
3018
- ForceImmediateRepoll();
3019
- }
3020
- const bool pending_initial_metadata =
3021
- server_initial_metadata_ready_.has_value();
3022
- if (!pending_initial_metadata) {
3023
- server_initial_metadata_ready_.emplace(
3024
- server_initial_metadata_.receiver.Next());
3025
- }
3026
- Poll<NextResult<ServerMetadataHandle>> r =
3027
- (*server_initial_metadata_ready_)();
3028
- server_initial_metadata_ready_.reset();
3029
- if (auto* result = r.value_if_ready()) {
3030
- if (pending_initial_metadata) PublishInitialMetadata(result->value().get());
3031
- is_trailers_only_ = false;
3032
- } else {
3033
- if (pending_initial_metadata) {
3034
- ServerMetadata no_metadata{GetContext<Arena>()};
3035
- PublishInitialMetadata(&no_metadata);
3036
- }
3037
- is_trailers_only_ = true;
3038
- }
2933
+ client_to_server_messages_.sender.Close();
3039
2934
  if (auto* channelz_channel = channel()->channelz_node()) {
3040
2935
  if (trailing_metadata->get(GrpcStatusMetadata())
3041
2936
  .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
@@ -3044,13 +2939,7 @@ void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
3044
2939
  channelz_channel->RecordCallFailed();
3045
2940
  }
3046
2941
  }
3047
- if (auto* status_request =
3048
- absl::get_if<grpc_op::grpc_op_data::grpc_op_recv_status_on_client>(
3049
- &recv_status_on_client_)) {
3050
- PublishStatus(*status_request, std::move(trailing_metadata));
3051
- } else {
3052
- recv_status_on_client_ = std::move(trailing_metadata);
3053
- }
2942
+ server_trailing_metadata_.Set(std::move(trailing_metadata));
3054
2943
  }
3055
2944
 
3056
2945
  namespace {
@@ -3076,35 +2965,43 @@ std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
3076
2965
  }
3077
2966
  } // namespace
3078
2967
 
3079
- void ClientPromiseBasedCall::PublishStatus(
2968
+ void ClientPromiseBasedCall::StartRecvStatusOnClient(
2969
+ const Completion& completion,
3080
2970
  grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
3081
- ServerMetadataHandle trailing_metadata) {
3082
- const grpc_status_code status = trailing_metadata->get(GrpcStatusMetadata())
3083
- .value_or(GRPC_STATUS_UNKNOWN);
3084
- *op_args.status = status;
3085
- absl::string_view message_string;
3086
- if (Slice* message = trailing_metadata->get_pointer(GrpcMessageMetadata())) {
3087
- message_string = message->as_string_view();
3088
- *op_args.status_details = message->Ref().TakeCSlice();
3089
- } else {
3090
- *op_args.status_details = grpc_empty_slice();
3091
- }
3092
- if (message_string.empty()) {
3093
- RunFinalization(status, nullptr);
3094
- } else {
3095
- std::string error_string(message_string);
3096
- RunFinalization(status, error_string.c_str());
3097
- }
3098
- if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
3099
- *op_args.error_string =
3100
- gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
3101
- }
3102
- PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata);
3103
- // Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding
3104
- // (so we don't call through twice)
3105
- recv_status_on_client_ = absl::monostate();
3106
- FinishOpOnCompletion(&recv_status_on_client_completion_,
3107
- PendingOp::kReceiveStatusOnClient);
2971
+ Party::BulkSpawner& spawner) {
2972
+ ForceCompletionSuccess(completion);
2973
+ spawner.Spawn(
2974
+ "recv_status_on_client", server_trailing_metadata_.Wait(),
2975
+ [this, op_args,
2976
+ completion =
2977
+ AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient)](
2978
+ ServerMetadataHandle trailing_metadata) mutable {
2979
+ const grpc_status_code status =
2980
+ trailing_metadata->get(GrpcStatusMetadata())
2981
+ .value_or(GRPC_STATUS_UNKNOWN);
2982
+ *op_args.status = status;
2983
+ absl::string_view message_string;
2984
+ if (Slice* message =
2985
+ trailing_metadata->get_pointer(GrpcMessageMetadata())) {
2986
+ message_string = message->as_string_view();
2987
+ *op_args.status_details = message->Ref().TakeCSlice();
2988
+ } else {
2989
+ *op_args.status_details = grpc_empty_slice();
2990
+ }
2991
+ if (message_string.empty()) {
2992
+ RunFinalization(status, nullptr);
2993
+ } else {
2994
+ std::string error_string(message_string);
2995
+ RunFinalization(status, error_string.c_str());
2996
+ }
2997
+ if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
2998
+ *op_args.error_string =
2999
+ gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
3000
+ }
3001
+ PublishMetadataArray(trailing_metadata.get(),
3002
+ op_args.trailing_metadata);
3003
+ FinishOpOnCompletion(&completion, PendingOp::kReceiveStatusOnClient);
3004
+ });
3108
3005
  }
3109
3006
  #endif
3110
3007
 
@@ -3117,19 +3014,18 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
3117
3014
  public:
3118
3015
  ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
3119
3016
 
3120
- void CancelWithErrorLocked(grpc_error_handle) override
3121
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3017
+ void CancelWithError(grpc_error_handle) override;
3122
3018
  grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
3123
3019
  bool is_notify_tag_closure) override;
3124
- bool failed_before_recv_message() const override { abort(); }
3020
+ bool failed_before_recv_message() const override { return false; }
3125
3021
  bool is_trailers_only() const override { abort(); }
3126
3022
  absl::string_view GetServerAuthority() const override { return ""; }
3127
3023
 
3128
3024
  // Polling order for the server promise stack:
3129
3025
  //
3130
3026
  // │ ┌───────────────────────────────────────┐
3131
- // │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management,
3132
- // │ ├───────────────────────────────────────┤ signal call end to app
3027
+ // │ │ ServerPromiseBasedCall ├──► Lifetime management
3028
+ // │ ├───────────────────────────────────────┤
3133
3029
  // │ │ ConnectedChannel ├─┐
3134
3030
  // │ ├───────────────────────────────────────┤ └► Interactions with the
3135
3031
  // │ │ ... closest to transport filter │ transport - send/recv msgs
@@ -3140,16 +3036,12 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
3140
3036
  // │ ├───────────────────────────────────────┤ │ setup, publishing call to
3141
3037
  // │ │ Server::ChannelData::MakeCallPromise ├─┘ application
3142
3038
  // │ ├───────────────────────────────────────┤
3143
- // │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions,
3144
- // ▼ └───────────────────────────────────────┘ forwarding messages,
3145
- // Polling & sending trailing metadata
3039
+ // │ │ MakeTopOfServerCallPromise ├──► Send trailing metadata
3040
+ // ▼ └───────────────────────────────────────┘
3041
+ // Polling &
3146
3042
  // instantiation
3147
3043
  // order
3148
3044
 
3149
- void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
3150
- Poll<ServerMetadataHandle> PollTopOfCall()
3151
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3152
-
3153
3045
  std::string DebugTag() const override {
3154
3046
  return absl::StrFormat("SERVER_CALL[%p]: ", this);
3155
3047
  }
@@ -3159,44 +3051,64 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
3159
3051
  private:
3160
3052
  class RecvCloseOpCancelState {
3161
3053
  public:
3162
- // Request that receiver be filled in per grpc_op_recv_close_on_server.
3163
- // Returns true if the request can be fulfilled immediately.
3164
- // Returns false if the request will be fulfilled later.
3054
+ // Request that receiver be filled in per
3055
+ // grpc_op_recv_close_on_server. Returns true if the request can
3056
+ // be fulfilled immediately. Returns false if the request will be
3057
+ // fulfilled later.
3165
3058
  bool ReceiveCloseOnServerOpStarted(int* receiver) {
3166
- switch (state_) {
3167
- case kUnset:
3168
- state_ = reinterpret_cast<uintptr_t>(receiver);
3169
- return false;
3170
- case kFinishedWithFailure:
3171
- *receiver = 1;
3172
- return true;
3173
- case kFinishedWithSuccess:
3174
- *receiver = 0;
3175
- return true;
3176
- default:
3177
- abort(); // unreachable
3178
- }
3059
+ uintptr_t state = state_.load(std::memory_order_acquire);
3060
+ uintptr_t new_state;
3061
+ do {
3062
+ switch (state) {
3063
+ case kUnset:
3064
+ new_state = reinterpret_cast<uintptr_t>(receiver);
3065
+ break;
3066
+ case kFinishedWithFailure:
3067
+ *receiver = 1;
3068
+ return true;
3069
+ case kFinishedWithSuccess:
3070
+ *receiver = 0;
3071
+ return true;
3072
+ default:
3073
+ Crash("Two threads offered ReceiveCloseOnServerOpStarted");
3074
+ }
3075
+ } while (!state_.compare_exchange_weak(state, new_state,
3076
+ std::memory_order_acq_rel,
3077
+ std::memory_order_acquire));
3078
+ return false;
3179
3079
  }
3180
3080
 
3181
3081
  // Mark the call as having completed.
3182
- // Returns true if this finishes a previous RequestReceiveCloseOnServer.
3183
- bool CompleteCall(bool success) {
3184
- switch (state_) {
3185
- case kUnset:
3186
- state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
3187
- return false;
3188
- case kFinishedWithFailure:
3189
- case kFinishedWithSuccess:
3190
- abort(); // unreachable
3191
- default:
3192
- *reinterpret_cast<int*>(state_) = success ? 0 : 1;
3193
- state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
3194
- return true;
3195
- }
3082
+ // Returns true if this finishes a previous
3083
+ // RequestReceiveCloseOnServer.
3084
+ bool CompleteCallWithCancelledSetTo(bool cancelled) {
3085
+ uintptr_t state = state_.load(std::memory_order_acquire);
3086
+ uintptr_t new_state;
3087
+ bool r;
3088
+ do {
3089
+ switch (state) {
3090
+ case kUnset:
3091
+ new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
3092
+ r = false;
3093
+ break;
3094
+ case kFinishedWithFailure:
3095
+ return false;
3096
+ case kFinishedWithSuccess:
3097
+ Crash("unreachable");
3098
+ default:
3099
+ new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
3100
+ r = true;
3101
+ }
3102
+ } while (!state_.compare_exchange_weak(state, new_state,
3103
+ std::memory_order_acq_rel,
3104
+ std::memory_order_acquire));
3105
+ if (r) *reinterpret_cast<int*>(state) = cancelled ? 1 : 0;
3106
+ return r;
3196
3107
  }
3197
3108
 
3198
3109
  std::string ToString() const {
3199
- switch (state_) {
3110
+ auto state = state_.load(std::memory_order_relaxed);
3111
+ switch (state) {
3200
3112
  case kUnset:
3201
3113
  return "Unset";
3202
3114
  case kFinishedWithFailure:
@@ -3205,7 +3117,7 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
3205
3117
  return "FinishedWithSuccess";
3206
3118
  default:
3207
3119
  return absl::StrFormat("WaitingForReceiver(%p)",
3208
- reinterpret_cast<void*>(state_));
3120
+ reinterpret_cast<void*>(state));
3209
3121
  }
3210
3122
  }
3211
3123
 
@@ -3213,37 +3125,28 @@ class ServerPromiseBasedCall final : public PromiseBasedCall {
3213
3125
  static constexpr uintptr_t kUnset = 0;
3214
3126
  static constexpr uintptr_t kFinishedWithFailure = 1;
3215
3127
  static constexpr uintptr_t kFinishedWithSuccess = 2;
3216
- // Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess
3217
- // OR an int* that wants to receive the final status.
3218
- uintptr_t state_ = kUnset;
3128
+ // Holds one of kUnset, kFinishedWithFailure, or
3129
+ // kFinishedWithSuccess OR an int* that wants to receive the
3130
+ // final status.
3131
+ std::atomic<uintptr_t> state_{kUnset};
3219
3132
  };
3220
3133
 
3221
3134
  grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
3222
3135
  void CommitBatch(const grpc_op* ops, size_t nops,
3223
- const Completion& completion)
3224
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3136
+ const Completion& completion);
3137
+ void Finish(ServerMetadataHandle result);
3225
3138
 
3226
3139
  friend class ServerCallContext;
3227
3140
  ServerCallContext call_context_;
3228
3141
  Server* const server_;
3229
- ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
3230
- PipeSender<MessageHandle>* server_to_client_messages_ ABSL_GUARDED_BY(mu()) =
3231
- nullptr;
3232
- PipeReceiver<MessageHandle>* client_to_server_messages_
3233
- ABSL_GUARDED_BY(mu()) = nullptr;
3234
- using SendInitialMetadataState =
3235
- absl::variant<absl::monostate, PipeSender<ServerMetadataHandle>*,
3236
- typename PipeSender<ServerMetadataHandle>::PushType>;
3237
- SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) =
3238
- absl::monostate{};
3239
- ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu());
3240
- grpc_compression_algorithm incoming_compression_algorithm_
3241
- ABSL_GUARDED_BY(mu());
3242
- RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu());
3243
- Completion recv_close_completion_ ABSL_GUARDED_BY(mu());
3244
- bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false;
3245
- Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu());
3246
- ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu());
3142
+ PipeSender<ServerMetadataHandle>* server_initial_metadata_ = nullptr;
3143
+ PipeSender<MessageHandle>* server_to_client_messages_ = nullptr;
3144
+ PipeReceiver<MessageHandle>* client_to_server_messages_ = nullptr;
3145
+ Latch<ServerMetadataHandle> send_trailing_metadata_;
3146
+ RecvCloseOpCancelState recv_close_op_cancel_state_;
3147
+ ClientMetadataHandle client_initial_metadata_;
3148
+ Completion recv_close_completion_;
3149
+ std::atomic<bool> cancelled_{false};
3247
3150
  };
3248
3151
 
3249
3152
  ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
@@ -3256,106 +3159,60 @@ ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
3256
3159
  if (channelz_node != nullptr) {
3257
3160
  channelz_node->RecordCallStarted();
3258
3161
  }
3259
- MutexLock lock(mu());
3162
+ // TODO(yashykt): In the future, we want to also enable stats and trace
3163
+ // collecting from when the call is created at the transport. The idea is that
3164
+ // the transport would create the call tracer and pass it in as part of the
3165
+ // metadata.
3166
+ auto* server_call_tracer_factory =
3167
+ ServerCallTracerFactory::Get(args->server->channel_args());
3168
+ if (server_call_tracer_factory != nullptr) {
3169
+ auto* server_call_tracer =
3170
+ server_call_tracer_factory->CreateNewServerCallTracer(arena);
3171
+ if (server_call_tracer != nullptr) {
3172
+ // Note that we are setting both
3173
+ // GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
3174
+ // GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
3175
+ // promise-based world, we would just a single tracer object for each
3176
+ // stack (call, subchannel_call, server_call.)
3177
+ ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
3178
+ server_call_tracer, nullptr);
3179
+ ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
3180
+ }
3181
+ }
3260
3182
  ScopedContext activity_context(this);
3261
- promise_ = channel()->channel_stack()->MakeServerCallPromise(
3262
- CallArgs{nullptr, nullptr, nullptr, nullptr});
3183
+ Spawn("server_promise",
3184
+ channel()->channel_stack()->MakeServerCallPromise(
3185
+ CallArgs{nullptr, ClientInitialMetadataOutstandingToken::Empty(),
3186
+ nullptr, nullptr, nullptr}),
3187
+ [this](ServerMetadataHandle result) { Finish(std::move(result)); });
3263
3188
  }
3264
3189
 
3265
- Poll<ServerMetadataHandle> ServerPromiseBasedCall::PollTopOfCall() {
3190
+ void ServerPromiseBasedCall::Finish(ServerMetadataHandle result) {
3266
3191
  if (grpc_call_trace.enabled()) {
3267
- gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(),
3268
- cancel_send_and_receive_ ? "force-" : "",
3269
- send_trailing_metadata_ != nullptr
3270
- ? absl::StrCat("send-metadata:",
3271
- send_trailing_metadata_->DebugString(), " ")
3272
- .c_str()
3273
- : " ",
3274
- PollStateDebugString().c_str());
3192
+ gpr_log(GPR_INFO, "%s[call] Finish: recv_close_state:%s result:%s",
3193
+ DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
3194
+ result->DebugString().c_str());
3275
3195
  }
3276
-
3277
- if (cancel_send_and_receive_) {
3278
- CancelSendMessage();
3279
- CancelRecvMessage();
3280
- }
3281
-
3282
- PollSendMessage();
3283
- PollRecvMessage(incoming_compression_algorithm_);
3284
-
3285
- if (!is_sending() && send_trailing_metadata_ != nullptr) {
3286
- server_to_client_messages_->Close();
3287
- return std::move(send_trailing_metadata_);
3196
+ if (recv_close_op_cancel_state_.CompleteCallWithCancelledSetTo(
3197
+ result->get(GrpcCallWasCancelled()).value_or(true))) {
3198
+ FinishOpOnCompletion(&recv_close_completion_,
3199
+ PendingOp::kReceiveCloseOnServer);
3288
3200
  }
3289
-
3290
- return Pending{};
3291
- }
3292
-
3293
- void ServerPromiseBasedCall::UpdateOnce() {
3294
- if (grpc_call_trace.enabled()) {
3295
- gpr_log(
3296
- GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s",
3297
- DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
3298
- recv_close_completion_.has_value()
3299
- ? absl::StrCat(":", CompletionString(recv_close_completion_))
3300
- .c_str()
3301
- : "",
3302
- send_status_from_server_completion_.has_value()
3303
- ? absl::StrCat(
3304
- "send_status:",
3305
- CompletionString(send_status_from_server_completion_), " ")
3306
- .c_str()
3307
- : "",
3308
- PollStateDebugString().c_str(),
3309
- promise_.has_value() ? "true" : "false");
3310
- }
3311
- if (auto* p =
3312
- absl::get_if<typename PipeSender<ServerMetadataHandle>::PushType>(
3313
- &send_initial_metadata_state_)) {
3314
- if ((*p)().ready()) {
3315
- send_initial_metadata_state_ = absl::monostate{};
3316
- }
3201
+ if (server_initial_metadata_ != nullptr) {
3202
+ server_initial_metadata_->Close();
3317
3203
  }
3318
- if (promise_.has_value()) {
3319
- auto r = promise_();
3320
- if (grpc_call_trace.enabled()) {
3321
- gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
3322
- DebugTag().c_str(),
3323
- PollToString(r, [](const ServerMetadataHandle& h) {
3324
- return h->DebugString();
3325
- }).c_str());
3326
- }
3327
- if (auto* result = r.value_if_ready()) {
3328
- if (grpc_call_trace.enabled()) {
3329
- gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s",
3330
- DebugTag().c_str(),
3331
- recv_close_op_cancel_state_.ToString().c_str(),
3332
- (*result)->DebugString().c_str());
3333
- }
3334
- if (recv_close_op_cancel_state_.CompleteCall(
3335
- (*result)->get(GrpcStatusFromWire()).value_or(false))) {
3336
- FinishOpOnCompletion(&recv_close_completion_,
3337
- PendingOp::kReceiveCloseOnServer);
3338
- }
3339
- channelz::ServerNode* channelz_node = server_->channelz_node();
3340
- if (channelz_node != nullptr) {
3341
- if ((*result)
3342
- ->get(GrpcStatusMetadata())
3343
- .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
3344
- channelz_node->RecordCallSucceeded();
3345
- } else {
3346
- channelz_node->RecordCallFailed();
3347
- }
3348
- }
3349
- if (send_status_from_server_completion_.has_value()) {
3350
- FinishOpOnCompletion(&send_status_from_server_completion_,
3351
- PendingOp::kSendStatusFromServer);
3352
- }
3353
- CancelSendMessage();
3354
- CancelRecvMessage();
3355
- set_completed();
3356
- promise_ = ArenaPromise<ServerMetadataHandle>();
3204
+ channelz::ServerNode* channelz_node = server_->channelz_node();
3205
+ if (channelz_node != nullptr) {
3206
+ if (result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN) ==
3207
+ GRPC_STATUS_OK) {
3208
+ channelz_node->RecordCallSucceeded();
3209
+ } else {
3210
+ channelz_node->RecordCallFailed();
3357
3211
  }
3358
3212
  }
3213
+ set_completed();
3214
+ ResetDeadline();
3215
+ PropagateCancellationToChildren();
3359
3216
  }
3360
3217
 
3361
3218
  grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
@@ -3396,56 +3253,92 @@ grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
3396
3253
 
3397
3254
  void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
3398
3255
  const Completion& completion) {
3256
+ Party::BulkSpawner spawner(this);
3399
3257
  for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3400
3258
  const grpc_op& op = ops[op_idx];
3401
3259
  switch (op.op) {
3402
3260
  case GRPC_OP_SEND_INITIAL_METADATA: {
3403
- // compression not implemented
3404
- GPR_ASSERT(
3405
- !op.data.send_initial_metadata.maybe_compression_level.is_set);
3406
- if (!completed()) {
3407
- auto metadata = arena()->MakePooled<ServerMetadata>(arena());
3408
- CToMetadata(op.data.send_initial_metadata.metadata,
3409
- op.data.send_initial_metadata.count, metadata.get());
3410
- if (grpc_call_trace.enabled()) {
3411
- gpr_log(GPR_INFO, "%s[call] Send initial metadata",
3412
- DebugTag().c_str());
3413
- }
3414
- auto* pipe = absl::get<PipeSender<ServerMetadataHandle>*>(
3415
- send_initial_metadata_state_);
3416
- send_initial_metadata_state_ = pipe->Push(std::move(metadata));
3261
+ auto metadata = arena()->MakePooled<ServerMetadata>(arena());
3262
+ PrepareOutgoingInitialMetadata(op, *metadata);
3263
+ CToMetadata(op.data.send_initial_metadata.metadata,
3264
+ op.data.send_initial_metadata.count, metadata.get());
3265
+ if (grpc_call_trace.enabled()) {
3266
+ gpr_log(GPR_INFO, "%s[call] Send initial metadata",
3267
+ DebugTag().c_str());
3417
3268
  }
3269
+ QueueSend();
3270
+ spawner.Spawn(
3271
+ "call_send_initial_metadata",
3272
+ [this, metadata = std::move(metadata)]() mutable {
3273
+ EnactSend();
3274
+ return server_initial_metadata_->Push(std::move(metadata));
3275
+ },
3276
+ [this,
3277
+ completion = AddOpToCompletion(
3278
+ completion, PendingOp::kSendInitialMetadata)](bool r) mutable {
3279
+ if (!r) FailCompletion(completion);
3280
+ FinishOpOnCompletion(&completion,
3281
+ PendingOp::kSendInitialMetadata);
3282
+ });
3418
3283
  } break;
3419
3284
  case GRPC_OP_SEND_MESSAGE:
3420
- StartSendMessage(op, completion, server_to_client_messages_);
3285
+ StartSendMessage(op, completion, server_to_client_messages_, spawner);
3421
3286
  break;
3422
3287
  case GRPC_OP_RECV_MESSAGE:
3423
- StartRecvMessage(op, completion, client_to_server_messages_);
3288
+ if (cancelled_.load(std::memory_order_relaxed)) {
3289
+ FailCompletion(completion);
3290
+ break;
3291
+ }
3292
+ StartRecvMessage(
3293
+ op, completion, []() { return []() { return Empty{}; }; },
3294
+ client_to_server_messages_, spawner);
3424
3295
  break;
3425
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
3426
- send_trailing_metadata_ = arena()->MakePooled<ServerMetadata>(arena());
3296
+ case GRPC_OP_SEND_STATUS_FROM_SERVER: {
3297
+ auto metadata = arena()->MakePooled<ServerMetadata>(arena());
3427
3298
  CToMetadata(op.data.send_status_from_server.trailing_metadata,
3428
3299
  op.data.send_status_from_server.trailing_metadata_count,
3429
- send_trailing_metadata_.get());
3430
- send_trailing_metadata_->Set(GrpcStatusMetadata(),
3431
- op.data.send_status_from_server.status);
3300
+ metadata.get());
3301
+ metadata->Set(GrpcStatusMetadata(),
3302
+ op.data.send_status_from_server.status);
3432
3303
  if (auto* details = op.data.send_status_from_server.status_details) {
3433
- send_trailing_metadata_->Set(GrpcMessageMetadata(),
3434
- Slice(CSliceRef(*details)));
3304
+ metadata->Set(GrpcMessageMetadata(), Slice(CSliceRef(*details)));
3435
3305
  }
3436
- send_status_from_server_completion_ =
3437
- AddOpToCompletion(completion, PendingOp::kSendStatusFromServer);
3438
- break;
3306
+ spawner.Spawn(
3307
+ "call_send_status_from_server",
3308
+ [this, metadata = std::move(metadata)]() mutable {
3309
+ bool r = true;
3310
+ if (send_trailing_metadata_.is_set()) {
3311
+ r = false;
3312
+ } else {
3313
+ send_trailing_metadata_.Set(std::move(metadata));
3314
+ }
3315
+ return Map(WaitForSendingStarted(), [this, r](Empty) {
3316
+ server_initial_metadata_->Close();
3317
+ server_to_client_messages_->Close();
3318
+ return r;
3319
+ });
3320
+ },
3321
+ [this, completion = AddOpToCompletion(
3322
+ completion, PendingOp::kSendStatusFromServer)](
3323
+ bool ok) mutable {
3324
+ if (!ok) FailCompletion(completion);
3325
+ FinishOpOnCompletion(&completion,
3326
+ PendingOp::kSendStatusFromServer);
3327
+ });
3328
+ } break;
3439
3329
  case GRPC_OP_RECV_CLOSE_ON_SERVER:
3440
3330
  if (grpc_call_trace.enabled()) {
3441
3331
  gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
3442
3332
  DebugTag().c_str(),
3443
3333
  recv_close_op_cancel_state_.ToString().c_str());
3444
3334
  }
3445
- if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
3335
+ ForceCompletionSuccess(completion);
3336
+ recv_close_completion_ =
3337
+ AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
3338
+ if (recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
3446
3339
  op.data.recv_close_on_server.cancelled)) {
3447
- recv_close_completion_ =
3448
- AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
3340
+ FinishOpOnCompletion(&recv_close_completion_,
3341
+ PendingOp::kReceiveCloseOnServer);
3449
3342
  }
3450
3343
  break;
3451
3344
  case GRPC_OP_RECV_STATUS_ON_CLIENT:
@@ -3460,8 +3353,6 @@ grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
3460
3353
  size_t nops,
3461
3354
  void* notify_tag,
3462
3355
  bool is_notify_tag_closure) {
3463
- MutexLock lock(mu());
3464
- ScopedContext activity_context(this);
3465
3356
  if (nops == 0) {
3466
3357
  EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
3467
3358
  return GRPC_CALL_OK;
@@ -3473,18 +3364,30 @@ grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
3473
3364
  Completion completion =
3474
3365
  StartCompletion(notify_tag, is_notify_tag_closure, ops);
3475
3366
  CommitBatch(ops, nops, completion);
3476
- Update();
3477
3367
  FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
3478
3368
  return GRPC_CALL_OK;
3479
3369
  }
3480
3370
 
3481
- void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) {
3482
- if (!promise_.has_value()) return;
3483
- cancel_send_and_receive_ = true;
3484
- send_trailing_metadata_ = ServerMetadataFromStatus(error, arena());
3485
- ForceWakeup();
3371
+ void ServerPromiseBasedCall::CancelWithError(absl::Status error) {
3372
+ cancelled_.store(true, std::memory_order_relaxed);
3373
+ Spawn(
3374
+ "cancel_with_error",
3375
+ [this, error = std::move(error)]() {
3376
+ if (!send_trailing_metadata_.is_set()) {
3377
+ auto md = ServerMetadataFromStatus(error);
3378
+ md->Set(GrpcCallWasCancelled(), true);
3379
+ send_trailing_metadata_.Set(std::move(md));
3380
+ }
3381
+ if (server_to_client_messages_ != nullptr) {
3382
+ server_to_client_messages_->Close();
3383
+ }
3384
+ if (server_initial_metadata_ != nullptr) {
3385
+ server_initial_metadata_->Close();
3386
+ }
3387
+ return Empty{};
3388
+ },
3389
+ [](Empty) {});
3486
3390
  }
3487
-
3488
3391
  #endif
3489
3392
 
3490
3393
  #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
@@ -3493,24 +3396,19 @@ ServerCallContext::MakeTopOfServerCallPromise(
3493
3396
  CallArgs call_args, grpc_completion_queue* cq,
3494
3397
  grpc_metadata_array* publish_initial_metadata,
3495
3398
  absl::FunctionRef<void(grpc_call* call)> publish) {
3496
- call_->mu()->AssertHeld();
3497
- call_->SetCompletionQueueLocked(cq);
3399
+ call_->SetCompletionQueue(cq);
3498
3400
  call_->server_to_client_messages_ = call_args.server_to_client_messages;
3499
3401
  call_->client_to_server_messages_ = call_args.client_to_server_messages;
3500
- call_->send_initial_metadata_state_ = call_args.server_initial_metadata;
3501
- call_->incoming_compression_algorithm_ =
3502
- call_args.client_initial_metadata->get(GrpcEncodingMetadata())
3503
- .value_or(GRPC_COMPRESS_NONE);
3402
+ call_->server_initial_metadata_ = call_args.server_initial_metadata;
3504
3403
  call_->client_initial_metadata_ =
3505
3404
  std::move(call_args.client_initial_metadata);
3405
+ call_->ProcessIncomingInitialMetadata(*call_->client_initial_metadata_);
3506
3406
  PublishMetadataArray(call_->client_initial_metadata_.get(),
3507
3407
  publish_initial_metadata);
3508
3408
  call_->ExternalRef();
3509
3409
  publish(call_->c_ptr());
3510
- return [this]() {
3511
- call_->mu()->AssertHeld();
3512
- return call_->PollTopOfCall();
3513
- };
3410
+ return Seq(call_->server_to_client_messages_->AwaitClosed(),
3411
+ call_->send_trailing_metadata_.Wait());
3514
3412
  }
3515
3413
  #else
3516
3414
  ArenaPromise<ServerMetadataHandle>
@@ -3613,7 +3511,9 @@ uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
3613
3511
  }
3614
3512
 
3615
3513
  uint32_t grpc_call_test_only_get_encodings_accepted_by_peer(grpc_call* call) {
3616
- return grpc_core::Call::FromC(call)->test_only_encodings_accepted_by_peer();
3514
+ return grpc_core::Call::FromC(call)
3515
+ ->encodings_accepted_by_peer()
3516
+ .ToLegacyBitmask();
3617
3517
  }
3618
3518
 
3619
3519
  grpc_core::Arena* grpc_call_get_arena(grpc_call* call) {
@@ -3662,7 +3562,9 @@ uint8_t grpc_call_is_client(grpc_call* call) {
3662
3562
 
3663
3563
  grpc_compression_algorithm grpc_call_compression_for_level(
3664
3564
  grpc_call* call, grpc_compression_level level) {
3665
- return grpc_core::Call::FromC(call)->compression_for_level(level);
3565
+ return grpc_core::Call::FromC(call)
3566
+ ->encodings_accepted_by_peer()
3567
+ .CompressionAlgorithmForLevel(level);
3666
3568
  }
3667
3569
 
3668
3570
  bool grpc_call_is_trailers_only(const grpc_call* call) {