grpc 1.46.3 → 1.53.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of grpc might be problematic. Click here for more details.

Files changed (1803) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +419 -241
  3. data/include/grpc/byte_buffer.h +76 -1
  4. data/include/grpc/byte_buffer_reader.h +19 -1
  5. data/include/grpc/compression.h +2 -2
  6. data/include/grpc/event_engine/endpoint_config.h +11 -5
  7. data/include/grpc/event_engine/event_engine.h +129 -39
  8. data/include/grpc/event_engine/internal/memory_allocator_impl.h +1 -1
  9. data/include/grpc/event_engine/internal/slice_cast.h +67 -0
  10. data/include/grpc/event_engine/memory_allocator.h +1 -16
  11. data/include/grpc/event_engine/port.h +1 -1
  12. data/include/grpc/event_engine/slice.h +306 -0
  13. data/include/grpc/event_engine/slice_buffer.h +159 -0
  14. data/include/grpc/fork.h +25 -1
  15. data/include/grpc/grpc.h +4 -14
  16. data/include/grpc/grpc_posix.h +1 -1
  17. data/include/grpc/grpc_security.h +11 -0
  18. data/include/grpc/impl/codegen/atm.h +3 -71
  19. data/include/grpc/impl/codegen/atm_gcc_atomic.h +3 -67
  20. data/include/grpc/impl/codegen/atm_gcc_sync.h +3 -61
  21. data/include/grpc/impl/codegen/atm_windows.h +3 -108
  22. data/include/grpc/impl/codegen/byte_buffer.h +4 -78
  23. data/include/grpc/impl/codegen/byte_buffer_reader.h +4 -19
  24. data/include/grpc/impl/codegen/compression_types.h +5 -83
  25. data/include/grpc/impl/codegen/connectivity_state.h +5 -21
  26. data/include/grpc/impl/codegen/fork.h +4 -25
  27. data/include/grpc/impl/codegen/gpr_types.h +4 -35
  28. data/include/grpc/impl/codegen/grpc_types.h +5 -791
  29. data/include/grpc/impl/codegen/log.h +3 -86
  30. data/include/grpc/impl/codegen/port_platform.h +3 -699
  31. data/include/grpc/impl/codegen/propagation_bits.h +3 -28
  32. data/include/grpc/impl/codegen/slice.h +4 -107
  33. data/include/grpc/impl/codegen/status.h +4 -131
  34. data/include/grpc/impl/codegen/sync.h +3 -42
  35. data/include/grpc/impl/codegen/sync_abseil.h +3 -12
  36. data/include/grpc/impl/codegen/sync_custom.h +3 -14
  37. data/include/grpc/impl/codegen/sync_generic.h +3 -25
  38. data/include/grpc/impl/codegen/sync_posix.h +3 -28
  39. data/include/grpc/impl/codegen/sync_windows.h +3 -16
  40. data/include/grpc/impl/compression_types.h +109 -0
  41. data/include/grpc/impl/connectivity_state.h +47 -0
  42. data/include/grpc/impl/grpc_types.h +836 -0
  43. data/include/grpc/impl/propagation_bits.h +54 -0
  44. data/include/grpc/impl/slice_type.h +112 -0
  45. data/include/grpc/load_reporting.h +1 -1
  46. data/include/grpc/module.modulemap +5 -1
  47. data/include/grpc/slice.h +1 -1
  48. data/include/grpc/status.h +131 -1
  49. data/include/grpc/support/atm.h +70 -1
  50. data/include/grpc/support/atm_gcc_atomic.h +59 -1
  51. data/include/grpc/support/atm_gcc_sync.h +58 -1
  52. data/include/grpc/support/atm_windows.h +105 -1
  53. data/include/grpc/support/log.h +87 -1
  54. data/include/grpc/support/log_windows.h +1 -1
  55. data/include/grpc/support/port_platform.h +767 -1
  56. data/include/grpc/support/string_util.h +1 -1
  57. data/include/grpc/support/sync.h +35 -2
  58. data/include/grpc/support/sync_abseil.h +11 -1
  59. data/include/grpc/support/sync_custom.h +13 -1
  60. data/include/grpc/support/sync_generic.h +24 -1
  61. data/include/grpc/support/sync_posix.h +27 -1
  62. data/include/grpc/support/sync_windows.h +15 -1
  63. data/include/grpc/support/time.h +31 -6
  64. data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +148 -0
  65. data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +52 -0
  66. data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +29 -0
  67. data/src/core/ext/filters/census/grpc_context.cc +19 -17
  68. data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +67 -61
  69. data/src/core/ext/filters/channel_idle/channel_idle_filter.h +26 -6
  70. data/src/core/ext/filters/channel_idle/idle_filter_state.h +6 -4
  71. data/src/core/ext/filters/client_channel/backend_metric.cc +19 -12
  72. data/src/core/ext/filters/client_channel/backend_metric.h +21 -11
  73. data/src/core/ext/filters/client_channel/backup_poller.cc +34 -34
  74. data/src/core/ext/filters/client_channel/backup_poller.h +26 -28
  75. data/src/core/ext/filters/client_channel/channel_connectivity.cc +89 -44
  76. data/src/core/ext/filters/client_channel/client_channel.cc +401 -451
  77. data/src/core/ext/filters/client_channel/client_channel.h +68 -26
  78. data/src/core/ext/filters/client_channel/client_channel_channelz.cc +19 -23
  79. data/src/core/ext/filters/client_channel/client_channel_channelz.h +33 -23
  80. data/src/core/ext/filters/client_channel/client_channel_factory.cc +19 -43
  81. data/src/core/ext/filters/client_channel/client_channel_factory.h +11 -11
  82. data/src/core/ext/filters/client_channel/client_channel_plugin.cc +22 -47
  83. data/src/core/ext/filters/client_channel/client_channel_service_config.cc +153 -0
  84. data/src/core/ext/filters/client_channel/{resolver_result_parsing.h → client_channel_service_config.h} +33 -27
  85. data/src/core/ext/filters/client_channel/config_selector.cc +1 -0
  86. data/src/core/ext/filters/client_channel/config_selector.h +29 -23
  87. data/src/core/ext/filters/client_channel/connector.h +19 -10
  88. data/src/core/ext/filters/client_channel/dynamic_filters.cc +40 -54
  89. data/src/core/ext/filters/client_channel/dynamic_filters.h +20 -11
  90. data/src/core/ext/filters/client_channel/global_subchannel_pool.cc +2 -0
  91. data/src/core/ext/filters/client_channel/global_subchannel_pool.h +24 -21
  92. data/src/core/ext/filters/client_channel/health/health_check_client.cc +16 -4
  93. data/src/core/ext/filters/client_channel/health/health_check_client.h +5 -3
  94. data/src/core/ext/filters/client_channel/http_proxy.cc +100 -112
  95. data/src/core/ext/filters/client_channel/http_proxy.h +38 -31
  96. data/src/core/ext/filters/client_channel/lb_call_state_internal.h +39 -0
  97. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc +8 -5
  98. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.h +4 -3
  99. data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +49 -0
  100. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc +37 -16
  101. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h +18 -13
  102. data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +71 -131
  103. data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h +41 -24
  104. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +410 -375
  105. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h +26 -27
  106. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc +15 -3
  107. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.h +9 -7
  108. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc +18 -20
  109. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +27 -22
  110. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc +28 -19
  111. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +26 -25
  112. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc +338 -0
  113. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h +57 -0
  114. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric_internal.h +117 -0
  115. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc +1129 -0
  116. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h +94 -0
  117. data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +174 -180
  118. data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc +366 -407
  119. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +491 -405
  120. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h +19 -9
  121. data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +619 -634
  122. data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +261 -234
  123. data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +165 -143
  124. data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc +128 -0
  125. data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.h +71 -0
  126. data/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/weighted_round_robin.cc +972 -0
  127. data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +223 -220
  128. data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +214 -153
  129. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_attributes.cc +42 -0
  130. data/src/core/ext/filters/client_channel/lb_policy/xds/{xds.h → xds_attributes.h} +19 -15
  131. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h +3 -3
  132. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +178 -256
  133. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +199 -210
  134. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +382 -368
  135. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.cc +819 -0
  136. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_override_host.h +67 -0
  137. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_wrr_locality.cc +369 -0
  138. data/src/core/ext/filters/client_channel/local_subchannel_pool.cc +4 -0
  139. data/src/core/ext/filters/client_channel/local_subchannel_pool.h +22 -21
  140. data/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc +34 -19
  141. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +531 -163
  142. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +50 -44
  143. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +31 -30
  144. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +89 -84
  145. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +300 -209
  146. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +82 -61
  147. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +18 -18
  148. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +20 -19
  149. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +2 -0
  150. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h +22 -22
  151. data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +54 -22
  152. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +21 -34
  153. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +19 -5
  154. data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +135 -63
  155. data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +136 -68
  156. data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +29 -14
  157. data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +21 -27
  158. data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +368 -305
  159. data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.h +6 -4
  160. data/src/core/ext/filters/client_channel/retry_filter.cc +166 -163
  161. data/src/core/ext/filters/client_channel/retry_filter.h +4 -3
  162. data/src/core/ext/filters/client_channel/retry_service_config.cc +197 -229
  163. data/src/core/ext/filters/client_channel/retry_service_config.h +29 -26
  164. data/src/core/ext/filters/client_channel/retry_throttle.cc +28 -37
  165. data/src/core/ext/filters/client_channel/retry_throttle.h +36 -29
  166. data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +34 -17
  167. data/src/core/ext/filters/client_channel/subchannel.cc +332 -384
  168. data/src/core/ext/filters/client_channel/subchannel.h +112 -82
  169. data/src/core/ext/filters/client_channel/subchannel_interface_internal.h +38 -0
  170. data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc +17 -77
  171. data/src/core/ext/filters/client_channel/subchannel_pool_interface.h +41 -47
  172. data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +34 -108
  173. data/src/core/ext/filters/client_channel/subchannel_stream_client.h +20 -12
  174. data/src/core/ext/filters/deadline/deadline_filter.cc +94 -76
  175. data/src/core/ext/filters/deadline/deadline_filter.h +14 -12
  176. data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +44 -22
  177. data/src/core/ext/filters/fault_injection/fault_injection_filter.h +20 -11
  178. data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc +118 -0
  179. data/src/core/ext/filters/fault_injection/{service_config_parser.h → fault_injection_service_config_parser.h} +34 -13
  180. data/src/core/ext/filters/http/client/http_client_filter.cc +76 -54
  181. data/src/core/ext/filters/http/client/http_client_filter.h +36 -23
  182. data/src/core/ext/filters/http/client_authority_filter.cc +33 -38
  183. data/src/core/ext/filters/http/client_authority_filter.h +27 -24
  184. data/src/core/ext/filters/http/http_filters_plugin.cc +46 -55
  185. data/src/core/ext/filters/http/message_compress/compression_filter.cc +307 -0
  186. data/src/core/ext/filters/http/message_compress/compression_filter.h +139 -0
  187. data/src/core/ext/filters/http/server/http_server_filter.cc +111 -271
  188. data/src/core/ext/filters/http/server/http_server_filter.h +58 -24
  189. data/src/core/ext/filters/message_size/message_size_filter.cc +117 -137
  190. data/src/core/ext/filters/message_size/message_size_filter.h +34 -17
  191. data/src/core/ext/filters/rbac/rbac_filter.cc +29 -16
  192. data/src/core/ext/filters/rbac/rbac_filter.h +11 -3
  193. data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +736 -523
  194. data/src/core/ext/filters/rbac/rbac_service_config_parser.h +18 -7
  195. data/src/core/ext/filters/server_config_selector/server_config_selector.h +20 -8
  196. data/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +28 -12
  197. data/src/core/ext/filters/server_config_selector/server_config_selector_filter.h +4 -3
  198. data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +219 -0
  199. data/src/core/ext/filters/stateful_session/stateful_session_filter.h +66 -0
  200. data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc +82 -0
  201. data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +93 -0
  202. data/src/core/ext/transport/chttp2/alpn/alpn.cc +18 -18
  203. data/src/core/ext/transport/chttp2/alpn/alpn.h +24 -24
  204. data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +185 -241
  205. data/src/core/ext/transport/chttp2/client/chttp2_connector.h +38 -30
  206. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +204 -211
  207. data/src/core/ext/transport/chttp2/server/chttp2_server.h +26 -26
  208. data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +24 -24
  209. data/src/core/ext/transport/chttp2/transport/bin_decoder.h +35 -34
  210. data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +43 -38
  211. data/src/core/ext/transport/chttp2/transport/bin_encoder.h +35 -31
  212. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +801 -1067
  213. data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +28 -23
  214. data/src/core/ext/transport/chttp2/transport/context_list.cc +23 -19
  215. data/src/core/ext/transport/chttp2/transport/context_list.h +32 -29
  216. data/src/core/ext/transport/chttp2/transport/decode_huff.cc +251 -0
  217. data/src/core/ext/transport/chttp2/transport/decode_huff.h +971 -0
  218. data/src/core/ext/transport/chttp2/transport/flow_control.cc +350 -300
  219. data/src/core/ext/transport/chttp2/transport/flow_control.h +248 -325
  220. data/src/core/ext/transport/chttp2/transport/frame.h +22 -26
  221. data/src/core/ext/transport/chttp2/transport/frame_data.cc +85 -234
  222. data/src/core/ext/transport/chttp2/transport/frame_data.h +38 -61
  223. data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +41 -36
  224. data/src/core/ext/transport/chttp2/transport/frame_goaway.h +24 -22
  225. data/src/core/ext/transport/chttp2/transport/frame_ping.cc +31 -25
  226. data/src/core/ext/transport/chttp2/transport/frame_ping.h +25 -22
  227. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +35 -26
  228. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +24 -21
  229. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +44 -77
  230. data/src/core/ext/transport/chttp2/transport/frame_settings.h +27 -23
  231. data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +34 -33
  232. data/src/core/ext/transport/chttp2/transport/frame_window_update.h +24 -21
  233. data/src/core/ext/transport/chttp2/transport/hpack_constants.h +10 -4
  234. data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +221 -372
  235. data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +338 -158
  236. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +6 -1
  237. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +16 -5
  238. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +516 -751
  239. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +38 -27
  240. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +86 -76
  241. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +79 -49
  242. data/src/core/ext/transport/chttp2/transport/http2_settings.cc +21 -19
  243. data/src/core/ext/transport/chttp2/transport/http2_settings.h +39 -38
  244. data/src/core/ext/transport/chttp2/transport/http_trace.cc +19 -0
  245. data/src/core/ext/transport/chttp2/transport/http_trace.h +24 -0
  246. data/src/core/ext/transport/chttp2/transport/huffsyms.cc +20 -20
  247. data/src/core/ext/transport/chttp2/transport/huffsyms.h +21 -21
  248. data/src/core/ext/transport/chttp2/transport/internal.h +248 -296
  249. data/src/core/ext/transport/chttp2/transport/parsing.cc +363 -132
  250. data/src/core/ext/transport/chttp2/transport/stream_lists.cc +30 -30
  251. data/src/core/ext/transport/chttp2/transport/stream_map.cc +24 -24
  252. data/src/core/ext/transport/chttp2/transport/stream_map.h +34 -33
  253. data/src/core/ext/transport/chttp2/transport/varint.cc +19 -20
  254. data/src/core/ext/transport/chttp2/transport/varint.h +39 -34
  255. data/src/core/ext/transport/chttp2/transport/writing.cc +121 -94
  256. data/src/core/ext/transport/inproc/inproc_plugin.cc +17 -18
  257. data/src/core/ext/transport/inproc/inproc_transport.cc +227 -274
  258. data/src/core/ext/transport/inproc/inproc_transport.h +23 -21
  259. data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c +5 -5
  260. data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.h +30 -10
  261. data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c +6 -5
  262. data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.h +33 -8
  263. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c +8 -327
  264. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.h +83 -1723
  265. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.c +388 -0
  266. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump_shared.upb.h +1953 -0
  267. data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c +2 -2
  268. data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.h +12 -4
  269. data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c +6 -4
  270. data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.h +33 -4
  271. data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c +1 -1
  272. data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.h +6 -2
  273. data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c +1 -1
  274. data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.h +6 -2
  275. data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.h +6 -2
  276. data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.h +12 -4
  277. data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c +1 -1
  278. data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.h +6 -2
  279. data/src/core/ext/upb-generated/envoy/annotations/resource.upb.c +1 -1
  280. data/src/core/ext/upb-generated/envoy/annotations/resource.upb.h +7 -3
  281. data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c +14 -14
  282. data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.h +90 -30
  283. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +25 -19
  284. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +168 -34
  285. data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c +3 -3
  286. data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.h +18 -6
  287. data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c +35 -43
  288. data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.h +195 -116
  289. data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c +1 -1
  290. data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.h +6 -2
  291. data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c +1 -1
  292. data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.h +6 -2
  293. data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c +11 -11
  294. data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.h +84 -28
  295. data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c +35 -12
  296. data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.h +179 -14
  297. data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c +1 -1
  298. data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.h +6 -2
  299. data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c +25 -23
  300. data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.h +156 -48
  301. data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c +7 -7
  302. data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.h +42 -14
  303. data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c +1 -1
  304. data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.h +6 -2
  305. data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c +1 -1
  306. data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.h +6 -2
  307. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c +2 -2
  308. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.h +12 -4
  309. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c +12 -12
  310. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.h +78 -26
  311. data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c +23 -20
  312. data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.h +137 -69
  313. data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c +1 -1
  314. data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.h +6 -2
  315. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +23 -21
  316. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +132 -44
  317. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +1 -1
  318. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +6 -2
  319. data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c +2 -2
  320. data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.h +12 -4
  321. data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c +17 -2
  322. data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.h +67 -2
  323. data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c +1 -1
  324. data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.h +6 -2
  325. data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c +1 -1
  326. data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.h +6 -2
  327. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c +3 -3
  328. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.h +18 -6
  329. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c +15 -9
  330. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.h +73 -18
  331. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c +3 -3
  332. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.h +30 -10
  333. data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c +1 -1
  334. data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.h +6 -2
  335. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +39 -13
  336. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +194 -15
  337. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c +5 -5
  338. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.h +42 -14
  339. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +5 -3
  340. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +28 -2
  341. data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c +7 -4
  342. data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.h +36 -4
  343. data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c +1 -1
  344. data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.h +6 -2
  345. data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c +6 -6
  346. data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.h +48 -16
  347. data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c +8 -8
  348. data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.h +54 -18
  349. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +26 -11
  350. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +124 -14
  351. data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c +19 -15
  352. data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.h +78 -69
  353. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +168 -82
  354. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +887 -166
  355. data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c +3 -3
  356. data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.h +18 -6
  357. data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c +34 -15
  358. data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.h +161 -22
  359. data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c +3 -2
  360. data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.h +15 -2
  361. data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c +1 -1
  362. data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.h +6 -2
  363. data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c +2 -2
  364. data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.h +12 -4
  365. data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c +1 -1
  366. data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.h +6 -2
  367. data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c +1 -1
  368. data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.h +6 -2
  369. data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.c +47 -0
  370. data/src/core/ext/upb-generated/envoy/config/trace/v3/opentelemetry.upb.h +107 -0
  371. data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c +1 -1
  372. data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.h +6 -2
  373. data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c +2 -2
  374. data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.h +12 -4
  375. data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.c +1 -0
  376. data/src/core/ext/upb-generated/envoy/config/trace/v3/trace.upb.h +1 -0
  377. data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c +2 -2
  378. data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.h +12 -4
  379. data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c +7 -6
  380. data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.h +27 -14
  381. data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c +1 -1
  382. data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +6 -2
  383. data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c +2 -2
  384. data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.h +30 -10
  385. data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c +2 -2
  386. data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.h +18 -6
  387. data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c +11 -4
  388. data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +58 -4
  389. data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c +6 -3
  390. data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.h +29 -2
  391. data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.c +62 -0
  392. data/src/core/ext/upb-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h +179 -0
  393. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +38 -27
  394. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +207 -52
  395. data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.c +46 -0
  396. data/src/core/ext/upb-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h +98 -0
  397. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.c +54 -0
  398. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +188 -0
  399. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.c +111 -0
  400. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/common/v3/common.upb.h +444 -0
  401. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.c +56 -0
  402. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +192 -0
  403. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.c +46 -0
  404. data/src/core/ext/upb-generated/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h +98 -0
  405. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c +10 -8
  406. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.h +66 -14
  407. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c +3 -3
  408. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +18 -6
  409. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c +13 -11
  410. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +73 -23
  411. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c +2 -2
  412. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +12 -4
  413. data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c +1 -1
  414. data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.h +6 -2
  415. data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c +133 -16
  416. data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.h +656 -12
  417. data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c +2 -2
  418. data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.h +12 -4
  419. data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c +5 -5
  420. data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.h +30 -10
  421. data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c +1 -1
  422. data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.h +6 -2
  423. data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c +4 -4
  424. data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.h +24 -8
  425. data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.c +47 -0
  426. data/src/core/ext/upb-generated/envoy/type/matcher/v3/filter_state.upb.h +113 -0
  427. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +4 -4
  428. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +24 -8
  429. data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c +2 -2
  430. data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.h +12 -4
  431. data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c +1 -1
  432. data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.h +6 -2
  433. data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c +1 -1
  434. data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.h +6 -2
  435. data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c +1 -1
  436. data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.h +6 -2
  437. data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c +3 -3
  438. data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.h +18 -6
  439. data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.c +43 -0
  440. data/src/core/ext/upb-generated/envoy/type/matcher/v3/status_code_input.upb.h +114 -0
  441. data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c +1 -1
  442. data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.h +12 -4
  443. data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c +2 -2
  444. data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.h +12 -4
  445. data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c +3 -3
  446. data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.h +18 -6
  447. data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c +7 -7
  448. data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.h +42 -14
  449. data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c +5 -5
  450. data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.h +30 -10
  451. data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c +3 -3
  452. data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.h +18 -6
  453. data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c +1 -1
  454. data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.h +6 -2
  455. data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.h +12 -4
  456. data/src/core/ext/upb-generated/envoy/type/v3/range.upb.c +2 -2
  457. data/src/core/ext/upb-generated/envoy/type/v3/range.upb.h +18 -6
  458. data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.c +63 -0
  459. data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_strategy.upb.h +202 -0
  460. data/src/core/ext/upb-generated/envoy/type/v3/ratelimit_unit.upb.h +3 -1
  461. data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c +1 -1
  462. data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.h +6 -2
  463. data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.h +6 -2
  464. data/src/core/ext/upb-generated/google/api/annotations.upb.h +1 -1
  465. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c +11 -11
  466. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.h +66 -22
  467. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c +9 -9
  468. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h +72 -24
  469. data/src/core/ext/upb-generated/google/api/http.upb.c +3 -3
  470. data/src/core/ext/upb-generated/google/api/http.upb.h +18 -6
  471. data/src/core/ext/upb-generated/google/api/httpbody.upb.c +1 -1
  472. data/src/core/ext/upb-generated/google/api/httpbody.upb.h +6 -2
  473. data/src/core/ext/upb-generated/google/protobuf/any.upb.c +1 -1
  474. data/src/core/ext/upb-generated/google/protobuf/any.upb.h +6 -2
  475. data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c +44 -43
  476. data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.h +177 -55
  477. data/src/core/ext/upb-generated/google/protobuf/duration.upb.c +1 -1
  478. data/src/core/ext/upb-generated/google/protobuf/duration.upb.h +6 -2
  479. data/src/core/ext/upb-generated/google/protobuf/empty.upb.c +1 -1
  480. data/src/core/ext/upb-generated/google/protobuf/empty.upb.h +6 -2
  481. data/src/core/ext/upb-generated/google/protobuf/struct.upb.c +2 -2
  482. data/src/core/ext/upb-generated/google/protobuf/struct.upb.h +18 -6
  483. data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c +1 -1
  484. data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.h +6 -2
  485. data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c +6 -6
  486. data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.h +54 -18
  487. data/src/core/ext/upb-generated/google/rpc/status.upb.c +1 -1
  488. data/src/core/ext/upb-generated/google/rpc/status.upb.h +6 -2
  489. data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c +2 -2
  490. data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.h +24 -8
  491. data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.h +6 -2
  492. data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c +7 -7
  493. data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.h +60 -20
  494. data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c +1 -1
  495. data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.h +12 -4
  496. data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c +2 -2
  497. data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.h +12 -4
  498. data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c +6 -6
  499. data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.h +54 -18
  500. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c +2 -2
  501. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.h +12 -4
  502. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c +7 -7
  503. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.h +42 -14
  504. data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c +3 -3
  505. data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.h +23 -11
  506. data/src/core/ext/upb-generated/udpa/annotations/security.upb.c +1 -1
  507. data/src/core/ext/upb-generated/udpa/annotations/security.upb.h +7 -3
  508. data/src/core/ext/upb-generated/udpa/annotations/status.upb.h +7 -3
  509. data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c +1 -1
  510. data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.h +7 -3
  511. data/src/core/ext/upb-generated/validate/validate.upb.c +22 -22
  512. data/src/core/ext/upb-generated/validate/validate.upb.h +139 -47
  513. data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c +3 -3
  514. data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.h +23 -11
  515. data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c +1 -1
  516. data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.h +7 -3
  517. data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c +3 -3
  518. data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.h +27 -11
  519. data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c +1 -1
  520. data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.h +7 -3
  521. data/src/core/ext/upb-generated/xds/core/v3/authority.upb.c +1 -1
  522. data/src/core/ext/upb-generated/xds/core/v3/authority.upb.h +6 -2
  523. data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.c +47 -0
  524. data/src/core/ext/upb-generated/xds/core/v3/cidr.upb.h +107 -0
  525. data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c +2 -2
  526. data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.h +12 -4
  527. data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c +1 -1
  528. data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.h +6 -2
  529. data/src/core/ext/upb-generated/xds/core/v3/extension.upb.c +1 -1
  530. data/src/core/ext/upb-generated/xds/core/v3/extension.upb.h +6 -2
  531. data/src/core/ext/upb-generated/xds/core/v3/resource.upb.c +1 -1
  532. data/src/core/ext/upb-generated/xds/core/v3/resource.upb.h +6 -2
  533. data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c +1 -1
  534. data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.h +12 -4
  535. data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c +1 -1
  536. data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.h +6 -2
  537. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +3 -2
  538. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +15 -2
  539. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c +46 -0
  540. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.h +113 -0
  541. data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.c +46 -0
  542. data/src/core/ext/upb-generated/xds/type/matcher/v3/cel.upb.h +98 -0
  543. data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.c +62 -0
  544. data/src/core/ext/upb-generated/xds/type/matcher/v3/domain.upb.h +174 -0
  545. data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.c +36 -0
  546. data/src/core/ext/upb-generated/xds/type/matcher/v3/http_inputs.upb.h +74 -0
  547. data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.c +65 -0
  548. data/src/core/ext/upb-generated/xds/type/matcher/v3/ip.upb.h +191 -0
  549. data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c +6 -6
  550. data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.h +54 -18
  551. data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.c +127 -0
  552. data/src/core/ext/upb-generated/xds/type/matcher/v3/range.upb.h +474 -0
  553. data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c +2 -2
  554. data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.h +12 -4
  555. data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c +1 -1
  556. data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.h +12 -4
  557. data/src/core/ext/upb-generated/xds/type/v3/cel.upb.c +67 -0
  558. data/src/core/ext/upb-generated/xds/type/v3/cel.upb.h +214 -0
  559. data/src/core/ext/upb-generated/xds/type/v3/range.upb.c +64 -0
  560. data/src/core/ext/upb-generated/xds/type/v3/range.upb.h +208 -0
  561. data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c +1 -2
  562. data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.h +6 -2
  563. data/src/core/ext/upbdefs-generated/envoy/admin/v3/clusters.upbdefs.c +54 -53
  564. data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.c +67 -274
  565. data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump.upbdefs.h +0 -85
  566. data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.c +277 -0
  567. data/src/core/ext/upbdefs-generated/envoy/admin/v3/config_dump_shared.upbdefs.h +125 -0
  568. data/src/core/ext/upbdefs-generated/envoy/admin/v3/listeners.upbdefs.c +14 -11
  569. data/src/core/ext/upbdefs-generated/envoy/config/accesslog/v3/accesslog.upbdefs.c +107 -107
  570. data/src/core/ext/upbdefs-generated/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +199 -187
  571. data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c +232 -222
  572. data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.h +0 -5
  573. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.c +95 -75
  574. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/address.upbdefs.h +5 -0
  575. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/base.upbdefs.c +187 -183
  576. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c +81 -75
  577. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/protocol.upbdefs.c +190 -186
  578. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.c +7 -3
  579. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/socket_option.upbdefs.h +5 -0
  580. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/substitution_format_string.upbdefs.c +16 -16
  581. data/src/core/ext/upbdefs-generated/envoy/config/endpoint/v3/endpoint_components.upbdefs.c +71 -66
  582. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.c +164 -137
  583. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener.upbdefs.h +5 -0
  584. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/quic_config.upbdefs.c +16 -12
  585. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/udp_listener_config.upbdefs.c +39 -31
  586. data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.c +140 -129
  587. data/src/core/ext/upbdefs-generated/envoy/config/rbac/v3/rbac.upbdefs.h +5 -0
  588. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c +82 -74
  589. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.h +2 -2
  590. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +740 -667
  591. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.h +20 -0
  592. data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.c +150 -139
  593. data/src/core/ext/upbdefs-generated/envoy/config/tap/v3/common.upbdefs.h +5 -0
  594. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/datadog.upbdefs.c +16 -14
  595. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.c +48 -0
  596. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/opentelemetry.upbdefs.h +35 -0
  597. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/trace.upbdefs.c +16 -12
  598. data/src/core/ext/upbdefs-generated/envoy/config/trace/v3/zipkin.upbdefs.c +20 -18
  599. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c +46 -25
  600. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/router/v3/router.upbdefs.c +53 -42
  601. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c +60 -0
  602. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h +40 -0
  603. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +485 -467
  604. data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c +50 -0
  605. data/src/core/ext/upbdefs-generated/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h +35 -0
  606. data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c +21 -18
  607. data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +149 -145
  608. data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.c +156 -84
  609. data/src/core/ext/upbdefs-generated/envoy/service/discovery/v3/discovery.upbdefs.h +35 -0
  610. data/src/core/ext/upbdefs-generated/envoy/service/status/v3/csds.upbdefs.c +113 -113
  611. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.c +48 -0
  612. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/filter_state.upbdefs.h +35 -0
  613. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/regex.upbdefs.c +29 -28
  614. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.c +40 -0
  615. data/src/core/ext/upbdefs-generated/envoy/type/matcher/v3/status_code_input.upbdefs.h +40 -0
  616. data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.c +69 -0
  617. data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_strategy.upbdefs.h +40 -0
  618. data/src/core/ext/upbdefs-generated/envoy/type/v3/ratelimit_unit.upbdefs.c +10 -9
  619. data/src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c +92 -90
  620. data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.c +45 -0
  621. data/src/core/ext/upbdefs-generated/xds/core/v3/cidr.upbdefs.h +35 -0
  622. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.c +43 -0
  623. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/cel.upbdefs.h +35 -0
  624. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.c +51 -0
  625. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/domain.upbdefs.h +40 -0
  626. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.c +36 -0
  627. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/http_inputs.upbdefs.h +35 -0
  628. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.c +55 -0
  629. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/ip.upbdefs.h +40 -0
  630. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/matcher.upbdefs.c +9 -9
  631. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.c +71 -0
  632. data/src/core/ext/upbdefs-generated/xds/type/matcher/v3/range.upbdefs.h +60 -0
  633. data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.c +60 -0
  634. data/src/core/ext/upbdefs-generated/xds/type/v3/cel.upbdefs.h +40 -0
  635. data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.c +36 -0
  636. data/src/core/ext/upbdefs-generated/xds/type/v3/range.upbdefs.h +45 -0
  637. data/src/core/ext/upbdefs-generated/xds/type/v3/typed_struct.upbdefs.c +4 -7
  638. data/src/core/ext/xds/certificate_provider_store.cc +69 -5
  639. data/src/core/ext/xds/certificate_provider_store.h +22 -5
  640. data/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +17 -10
  641. data/src/core/ext/xds/file_watcher_certificate_provider_factory.h +12 -4
  642. data/src/core/ext/xds/upb_utils.h +3 -25
  643. data/src/core/ext/xds/xds_api.cc +90 -196
  644. data/src/core/ext/xds/xds_api.h +41 -39
  645. data/src/core/ext/xds/xds_bootstrap.cc +5 -537
  646. data/src/core/ext/xds/xds_bootstrap.h +41 -96
  647. data/src/core/ext/xds/xds_bootstrap_grpc.cc +362 -0
  648. data/src/core/ext/xds/xds_bootstrap_grpc.h +184 -0
  649. data/src/core/ext/xds/xds_certificate_provider.cc +35 -26
  650. data/src/core/ext/xds/xds_certificate_provider.h +29 -5
  651. data/src/core/ext/xds/xds_channel_args.h +3 -3
  652. data/src/core/ext/xds/xds_channel_stack_modifier.cc +17 -7
  653. data/src/core/ext/xds/xds_channel_stack_modifier.h +16 -4
  654. data/src/core/ext/xds/xds_client.cc +849 -1316
  655. data/src/core/ext/xds/xds_client.h +57 -61
  656. data/src/core/ext/xds/xds_client_grpc.cc +235 -0
  657. data/src/core/ext/xds/xds_client_grpc.h +79 -0
  658. data/src/core/ext/xds/xds_client_stats.cc +24 -25
  659. data/src/core/ext/xds/xds_client_stats.h +26 -24
  660. data/src/core/ext/xds/xds_cluster.cc +505 -247
  661. data/src/core/ext/xds/xds_cluster.h +69 -40
  662. data/src/core/ext/xds/xds_cluster_specifier_plugin.cc +67 -77
  663. data/src/core/ext/xds/xds_cluster_specifier_plugin.h +47 -29
  664. data/src/core/ext/xds/xds_common_types.cc +238 -123
  665. data/src/core/ext/xds/xds_common_types.h +29 -16
  666. data/src/core/ext/xds/xds_endpoint.cc +265 -162
  667. data/src/core/ext/xds/xds_endpoint.h +19 -15
  668. data/src/core/ext/xds/xds_health_status.cc +80 -0
  669. data/src/core/ext/xds/xds_health_status.h +109 -0
  670. data/src/core/ext/xds/xds_http_fault_filter.cc +67 -58
  671. data/src/core/ext/xds/xds_http_fault_filter.h +19 -25
  672. data/src/core/ext/xds/xds_http_filters.cc +70 -71
  673. data/src/core/ext/xds/xds_http_filters.h +73 -25
  674. data/src/core/ext/xds/xds_http_rbac_filter.cc +168 -225
  675. data/src/core/ext/xds/xds_http_rbac_filter.h +22 -18
  676. data/src/core/ext/xds/xds_http_stateful_session_filter.cc +218 -0
  677. data/src/core/ext/xds/xds_http_stateful_session_filter.h +58 -0
  678. data/src/core/ext/xds/xds_lb_policy_registry.cc +334 -0
  679. data/src/core/ext/xds/xds_lb_policy_registry.h +71 -0
  680. data/src/core/ext/xds/xds_listener.cc +573 -481
  681. data/src/core/ext/xds/xds_listener.h +57 -51
  682. data/src/core/ext/xds/xds_resource_type.h +28 -22
  683. data/src/core/ext/xds/xds_resource_type_impl.h +18 -17
  684. data/src/core/ext/xds/xds_route_config.cc +592 -570
  685. data/src/core/ext/xds/xds_route_config.h +74 -40
  686. data/src/core/ext/xds/xds_routing.cc +21 -7
  687. data/src/core/ext/xds/xds_routing.h +17 -12
  688. data/src/core/ext/xds/xds_server_config_fetcher.cc +227 -178
  689. data/src/core/ext/xds/xds_transport.h +86 -0
  690. data/src/core/ext/xds/xds_transport_grpc.cc +356 -0
  691. data/src/core/ext/xds/xds_transport_grpc.h +135 -0
  692. data/src/core/lib/address_utils/parse_address.cc +52 -52
  693. data/src/core/lib/address_utils/parse_address.h +46 -42
  694. data/src/core/lib/address_utils/sockaddr_utils.cc +90 -54
  695. data/src/core/lib/address_utils/sockaddr_utils.h +45 -40
  696. data/src/core/lib/avl/avl.h +57 -33
  697. data/src/core/lib/backoff/backoff.cc +19 -21
  698. data/src/core/lib/backoff/backoff.h +21 -21
  699. data/src/core/lib/backoff/random_early_detection.cc +31 -0
  700. data/src/core/lib/backoff/random_early_detection.h +59 -0
  701. data/src/core/lib/channel/call_finalization.h +6 -4
  702. data/src/core/lib/channel/call_tracer.h +23 -8
  703. data/src/core/lib/channel/channel_args.cc +149 -61
  704. data/src/core/lib/channel/channel_args.h +308 -104
  705. data/src/core/lib/channel/channel_args_preconditioning.cc +4 -3
  706. data/src/core/lib/channel/channel_args_preconditioning.h +6 -5
  707. data/src/core/lib/channel/channel_fwd.h +26 -0
  708. data/src/core/lib/channel/channel_stack.cc +92 -52
  709. data/src/core/lib/channel/channel_stack.h +184 -135
  710. data/src/core/lib/channel/channel_stack_builder.cc +26 -39
  711. data/src/core/lib/channel/channel_stack_builder.h +33 -40
  712. data/src/core/lib/channel/channel_stack_builder_impl.cc +84 -60
  713. data/src/core/lib/channel/channel_stack_builder_impl.h +9 -9
  714. data/src/core/lib/channel/channel_trace.cc +29 -36
  715. data/src/core/lib/channel/channel_trace.h +27 -24
  716. data/src/core/lib/channel/channelz.cc +50 -65
  717. data/src/core/lib/channel/channelz.h +50 -33
  718. data/src/core/lib/channel/channelz_registry.cc +28 -36
  719. data/src/core/lib/channel/channelz_registry.h +31 -30
  720. data/src/core/lib/channel/connected_channel.cc +1267 -78
  721. data/src/core/lib/channel/connected_channel.h +21 -23
  722. data/src/core/lib/channel/context.h +25 -21
  723. data/src/core/lib/channel/promise_based_filter.cc +1896 -377
  724. data/src/core/lib/channel/promise_based_filter.h +592 -92
  725. data/src/core/lib/channel/status_util.cc +64 -17
  726. data/src/core/lib/channel/status_util.h +38 -24
  727. data/src/core/lib/compression/compression.cc +24 -19
  728. data/src/core/lib/compression/compression_internal.cc +88 -63
  729. data/src/core/lib/compression/compression_internal.h +28 -27
  730. data/src/core/lib/compression/message_compress.cc +28 -26
  731. data/src/core/lib/compression/message_compress.h +28 -29
  732. data/src/core/lib/config/core_configuration.cc +8 -1
  733. data/src/core/lib/config/core_configuration.h +86 -39
  734. data/src/core/lib/debug/event_log.cc +88 -0
  735. data/src/core/lib/debug/event_log.h +81 -0
  736. data/src/core/lib/debug/histogram_view.cc +69 -0
  737. data/src/core/lib/debug/histogram_view.h +37 -0
  738. data/src/core/lib/debug/stats.cc +48 -152
  739. data/src/core/lib/debug/stats.h +50 -57
  740. data/src/core/lib/debug/stats_data.cc +302 -645
  741. data/src/core/lib/debug/stats_data.h +293 -545
  742. data/src/core/lib/debug/trace.cc +18 -20
  743. data/src/core/lib/debug/trace.h +27 -48
  744. data/src/core/lib/event_engine/channel_args_endpoint_config.cc +16 -17
  745. data/src/core/lib/event_engine/channel_args_endpoint_config.h +16 -9
  746. data/src/core/lib/event_engine/common_closures.h +71 -0
  747. data/src/core/lib/event_engine/default_event_engine.cc +99 -0
  748. data/src/core/lib/event_engine/default_event_engine.h +73 -0
  749. data/src/core/lib/event_engine/default_event_engine_factory.cc +24 -3
  750. data/src/core/lib/event_engine/{event_engine_factory.h → default_event_engine_factory.h} +6 -9
  751. data/src/core/lib/event_engine/event_engine.cc +3 -30
  752. data/src/core/lib/event_engine/executor/executor.h +38 -0
  753. data/src/core/lib/event_engine/forkable.cc +106 -0
  754. data/src/core/lib/event_engine/forkable.h +61 -0
  755. data/src/core/lib/event_engine/handle_containers.h +74 -0
  756. data/src/core/lib/event_engine/memory_allocator.cc +11 -3
  757. data/src/core/lib/event_engine/poller.h +62 -0
  758. data/src/core/lib/event_engine/posix.h +158 -0
  759. data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +642 -0
  760. data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.h +139 -0
  761. data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +899 -0
  762. data/src/core/lib/event_engine/posix_engine/ev_poll_posix.h +97 -0
  763. data/src/core/lib/event_engine/posix_engine/event_poller.h +111 -0
  764. data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +74 -0
  765. data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.h +33 -0
  766. data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +77 -0
  767. data/src/core/lib/event_engine/posix_engine/internal_errqueue.h +179 -0
  768. data/src/core/lib/event_engine/posix_engine/lockfree_event.cc +265 -0
  769. data/src/core/lib/event_engine/posix_engine/lockfree_event.h +73 -0
  770. data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +1305 -0
  771. data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +717 -0
  772. data/src/core/lib/event_engine/posix_engine/posix_engine.cc +640 -0
  773. data/src/core/lib/event_engine/posix_engine/posix_engine.h +259 -0
  774. data/src/core/lib/event_engine/posix_engine/posix_engine_closure.h +80 -0
  775. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +289 -0
  776. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.h +279 -0
  777. data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +379 -0
  778. data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.h +91 -0
  779. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +853 -0
  780. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +316 -0
  781. data/src/core/lib/event_engine/posix_engine/timer.cc +311 -0
  782. data/src/core/lib/event_engine/posix_engine/timer.h +194 -0
  783. data/src/core/lib/event_engine/posix_engine/timer_heap.cc +107 -0
  784. data/src/core/lib/event_engine/posix_engine/timer_heap.h +56 -0
  785. data/src/core/lib/event_engine/posix_engine/timer_manager.cc +173 -0
  786. data/src/core/lib/event_engine/posix_engine/timer_manager.h +114 -0
  787. data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +332 -0
  788. data/src/core/lib/event_engine/posix_engine/traced_buffer_list.h +185 -0
  789. data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc +127 -0
  790. data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.h +45 -0
  791. data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc +150 -0
  792. data/src/core/lib/{iomgr/event_engine/promise.h → event_engine/posix_engine/wakeup_fd_pipe.h} +19 -25
  793. data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h +76 -0
  794. data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc +67 -0
  795. data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h +37 -0
  796. data/src/core/lib/event_engine/resolved_address.cc +23 -2
  797. data/src/core/lib/{iomgr/event_engine → event_engine}/resolved_address_internal.h +5 -8
  798. data/src/core/lib/event_engine/shim.cc +56 -0
  799. data/src/core/lib/event_engine/shim.h +33 -0
  800. data/src/core/lib/event_engine/slice.cc +103 -0
  801. data/src/core/lib/event_engine/slice_buffer.cc +50 -0
  802. data/src/core/lib/event_engine/tcp_socket_utils.cc +389 -0
  803. data/src/core/lib/event_engine/tcp_socket_utils.h +90 -0
  804. data/src/core/lib/event_engine/thread_local.cc +29 -0
  805. data/src/core/lib/event_engine/thread_local.h +32 -0
  806. data/src/core/lib/event_engine/thread_pool.cc +253 -0
  807. data/src/core/lib/event_engine/thread_pool.h +141 -0
  808. data/src/core/lib/event_engine/time_util.cc +30 -0
  809. data/src/core/lib/event_engine/time_util.h +32 -0
  810. data/src/core/lib/{iomgr/endpoint_pair_event_engine.cc → event_engine/trace.cc} +9 -17
  811. data/src/core/lib/event_engine/trace.h +43 -0
  812. data/src/core/lib/event_engine/utils.cc +44 -0
  813. data/src/core/lib/event_engine/utils.h +44 -0
  814. data/src/core/lib/event_engine/windows/iocp.cc +140 -0
  815. data/src/core/lib/event_engine/windows/iocp.h +69 -0
  816. data/src/core/lib/event_engine/windows/win_socket.cc +219 -0
  817. data/src/core/lib/event_engine/windows/win_socket.h +129 -0
  818. data/src/core/lib/event_engine/windows/windows_endpoint.cc +331 -0
  819. data/src/core/lib/event_engine/windows/windows_endpoint.h +103 -0
  820. data/src/core/lib/event_engine/windows/windows_engine.cc +388 -0
  821. data/src/core/lib/event_engine/windows/windows_engine.h +163 -0
  822. data/src/core/lib/experiments/config.cc +161 -0
  823. data/src/core/lib/experiments/config.h +53 -0
  824. data/src/core/lib/experiments/experiments.cc +81 -0
  825. data/src/core/lib/experiments/experiments.h +117 -0
  826. data/src/core/lib/gpr/alloc.cc +19 -25
  827. data/src/core/lib/gpr/alloc.h +20 -20
  828. data/src/core/lib/gpr/atm.cc +17 -17
  829. data/src/core/lib/gpr/cpu_iphone.cc +24 -24
  830. data/src/core/lib/gpr/cpu_linux.cc +28 -23
  831. data/src/core/lib/gpr/cpu_posix.cc +23 -22
  832. data/src/core/lib/gpr/cpu_windows.cc +20 -18
  833. data/src/core/lib/gpr/log.cc +30 -17
  834. data/src/core/lib/gpr/log_android.cc +22 -20
  835. data/src/core/lib/gpr/log_linux.cc +24 -24
  836. data/src/core/lib/gpr/log_posix.cc +20 -19
  837. data/src/core/lib/gpr/log_windows.cc +25 -25
  838. data/src/core/lib/gpr/spinlock.h +20 -20
  839. data/src/core/lib/gpr/string.cc +25 -24
  840. data/src/core/lib/gpr/string.h +61 -61
  841. data/src/core/lib/gpr/string_posix.cc +24 -24
  842. data/src/core/lib/gpr/string_util_windows.cc +25 -52
  843. data/src/core/lib/gpr/string_windows.cc +24 -24
  844. data/src/core/lib/gpr/sync.cc +25 -25
  845. data/src/core/lib/gpr/sync_abseil.cc +36 -40
  846. data/src/core/lib/gpr/sync_posix.cc +22 -34
  847. data/src/core/lib/gpr/sync_windows.cc +29 -27
  848. data/src/core/lib/gpr/time.cc +34 -30
  849. data/src/core/lib/gpr/time_posix.cc +41 -45
  850. data/src/core/lib/gpr/time_precise.cc +22 -22
  851. data/src/core/lib/gpr/time_precise.h +21 -22
  852. data/src/core/lib/gpr/time_windows.cc +35 -29
  853. data/src/core/lib/gpr/tmpfile.h +24 -24
  854. data/src/core/lib/gpr/tmpfile_msys.cc +21 -20
  855. data/src/core/lib/gpr/tmpfile_posix.cc +22 -20
  856. data/src/core/lib/gpr/tmpfile_windows.cc +28 -29
  857. data/src/core/lib/gpr/useful.h +83 -30
  858. data/src/core/lib/gpr/wrap_memcpy.cc +23 -23
  859. data/src/core/lib/gprpp/atomic_utils.h +20 -20
  860. data/src/core/lib/gprpp/bitset.h +35 -17
  861. data/src/core/lib/gprpp/chunked_vector.h +7 -3
  862. data/src/core/lib/gprpp/construct_destruct.h +4 -3
  863. data/src/core/lib/gprpp/cpp_impl_of.h +3 -3
  864. data/src/core/{ext/xds/xds_resource_type.cc → lib/gprpp/crash.cc} +12 -12
  865. data/src/core/lib/gprpp/crash.h +34 -0
  866. data/src/core/lib/gprpp/debug_location.h +60 -31
  867. data/src/core/lib/gprpp/dual_ref_counted.h +4 -7
  868. data/src/core/lib/gprpp/env.h +53 -0
  869. data/src/core/lib/gprpp/env_linux.cc +80 -0
  870. data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_event_engine.cc → lib/gprpp/env_posix.cc} +25 -9
  871. data/src/core/lib/gprpp/env_windows.cc +56 -0
  872. data/src/core/lib/gprpp/examine_stack.cc +17 -17
  873. data/src/core/lib/gprpp/examine_stack.h +21 -22
  874. data/src/core/lib/gprpp/fork.cc +58 -53
  875. data/src/core/lib/gprpp/fork.h +29 -35
  876. data/src/core/lib/gprpp/global_config.h +22 -24
  877. data/src/core/lib/gprpp/global_config_custom.h +20 -20
  878. data/src/core/lib/gprpp/global_config_env.cc +28 -25
  879. data/src/core/lib/gprpp/global_config_env.h +29 -27
  880. data/src/core/lib/gprpp/global_config_generic.h +21 -25
  881. data/src/core/lib/gprpp/host_port.cc +30 -26
  882. data/src/core/lib/gprpp/host_port.h +32 -31
  883. data/src/core/lib/gprpp/load_file.cc +75 -0
  884. data/src/core/lib/gprpp/load_file.h +33 -0
  885. data/src/core/lib/gprpp/manual_constructor.h +21 -89
  886. data/src/core/lib/gprpp/match.h +5 -3
  887. data/src/core/lib/gprpp/memory.h +22 -26
  888. data/src/core/lib/gprpp/mpscq.cc +17 -17
  889. data/src/core/lib/gprpp/mpscq.h +21 -21
  890. data/src/core/lib/gprpp/no_destruct.h +95 -0
  891. data/src/core/lib/gprpp/notification.h +67 -0
  892. data/src/core/lib/gprpp/orphanable.h +22 -25
  893. data/src/core/lib/gprpp/overload.h +3 -3
  894. data/src/core/lib/gprpp/packed_table.h +40 -0
  895. data/src/core/lib/gprpp/per_cpu.h +46 -0
  896. data/src/core/lib/gprpp/ref_counted.h +22 -24
  897. data/src/core/lib/gprpp/ref_counted_ptr.h +42 -55
  898. data/src/core/lib/gprpp/single_set_ptr.h +3 -3
  899. data/src/core/lib/gprpp/sorted_pack.h +98 -0
  900. data/src/core/lib/gprpp/stat.h +3 -5
  901. data/src/core/lib/gprpp/stat_posix.cc +10 -4
  902. data/src/core/lib/gprpp/stat_windows.cc +4 -2
  903. data/src/core/lib/gprpp/status_helper.cc +55 -35
  904. data/src/core/lib/gprpp/status_helper.h +13 -5
  905. data/src/core/lib/gprpp/strerror.cc +43 -0
  906. data/src/core/lib/gprpp/strerror.h +29 -0
  907. data/src/core/lib/gprpp/sync.h +26 -24
  908. data/src/core/lib/gprpp/table.h +22 -4
  909. data/src/core/lib/gprpp/tchar.cc +49 -0
  910. data/src/core/lib/gprpp/tchar.h +33 -0
  911. data/src/core/lib/gprpp/thd.h +25 -28
  912. data/src/core/lib/gprpp/thd_posix.cc +33 -32
  913. data/src/core/lib/gprpp/thd_windows.cc +28 -26
  914. data/src/core/lib/gprpp/time.cc +52 -9
  915. data/src/core/lib/gprpp/time.h +78 -4
  916. data/src/core/lib/gprpp/time_averaged_stats.cc +60 -0
  917. data/src/core/lib/gprpp/time_averaged_stats.h +79 -0
  918. data/src/core/lib/gprpp/time_util.cc +4 -0
  919. data/src/core/lib/gprpp/time_util.h +3 -3
  920. data/src/core/lib/gprpp/unique_type_name.h +104 -0
  921. data/src/core/lib/gprpp/validation_errors.cc +61 -0
  922. data/src/core/lib/gprpp/validation_errors.h +127 -0
  923. data/src/core/lib/{iomgr → gprpp}/work_serializer.cc +34 -18
  924. data/src/core/lib/{iomgr → gprpp}/work_serializer.h +21 -28
  925. data/src/core/lib/handshaker/proxy_mapper.h +53 -0
  926. data/src/core/lib/handshaker/proxy_mapper_registry.cc +71 -0
  927. data/src/core/lib/handshaker/proxy_mapper_registry.h +75 -0
  928. data/src/core/lib/http/format_request.cc +53 -22
  929. data/src/core/lib/http/format_request.h +23 -21
  930. data/src/core/lib/http/httpcli.cc +139 -141
  931. data/src/core/lib/http/httpcli.h +86 -38
  932. data/src/core/lib/http/httpcli_security_connector.cc +50 -42
  933. data/src/core/lib/http/httpcli_ssl_credentials.h +6 -4
  934. data/src/core/lib/http/parser.cc +78 -90
  935. data/src/core/lib/http/parser.h +38 -35
  936. data/src/core/lib/iomgr/block_annotate.h +23 -23
  937. data/src/core/lib/iomgr/buffer_list.cc +156 -136
  938. data/src/core/lib/iomgr/buffer_list.h +123 -101
  939. data/src/core/lib/iomgr/call_combiner.cc +32 -64
  940. data/src/core/lib/iomgr/call_combiner.h +24 -25
  941. data/src/core/lib/iomgr/cfstream_handle.cc +33 -36
  942. data/src/core/lib/iomgr/cfstream_handle.h +25 -25
  943. data/src/core/lib/iomgr/{event_engine/pollset.h → closure.cc} +9 -7
  944. data/src/core/lib/iomgr/closure.h +95 -58
  945. data/src/core/lib/iomgr/combiner.cc +20 -39
  946. data/src/core/lib/iomgr/combiner.h +20 -20
  947. data/src/core/lib/iomgr/dualstack_socket_posix.cc +21 -21
  948. data/src/core/lib/iomgr/dynamic_annotations.h +22 -22
  949. data/src/core/lib/iomgr/endpoint.cc +21 -21
  950. data/src/core/lib/iomgr/endpoint.h +53 -51
  951. data/src/core/lib/iomgr/endpoint_cfstream.cc +51 -47
  952. data/src/core/lib/iomgr/endpoint_cfstream.h +32 -32
  953. data/src/core/lib/iomgr/endpoint_pair.h +22 -22
  954. data/src/core/lib/iomgr/endpoint_pair_posix.cc +36 -29
  955. data/src/core/lib/iomgr/endpoint_pair_windows.cc +31 -22
  956. data/src/core/lib/iomgr/error.cc +51 -834
  957. data/src/core/lib/iomgr/error.h +45 -317
  958. data/src/core/lib/iomgr/error_cfstream.cc +18 -23
  959. data/src/core/lib/iomgr/error_cfstream.h +21 -21
  960. data/src/core/lib/iomgr/ev_apple.cc +21 -21
  961. data/src/core/lib/iomgr/ev_apple.h +21 -21
  962. data/src/core/lib/iomgr/ev_epoll1_linux.cc +263 -248
  963. data/src/core/lib/iomgr/ev_epoll1_linux.h +21 -21
  964. data/src/core/lib/iomgr/ev_poll_posix.cc +296 -271
  965. data/src/core/lib/iomgr/ev_poll_posix.h +22 -22
  966. data/src/core/lib/iomgr/ev_posix.cc +88 -121
  967. data/src/core/lib/iomgr/ev_posix.h +93 -87
  968. data/src/core/lib/iomgr/ev_windows.cc +18 -18
  969. data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.cc +15 -30
  970. data/src/core/lib/iomgr/{event_engine → event_engine_shims}/closure.h +10 -13
  971. data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +430 -0
  972. data/src/core/lib/iomgr/event_engine_shims/endpoint.h +43 -0
  973. data/src/core/lib/iomgr/event_engine_shims/tcp_client.cc +91 -0
  974. data/src/core/lib/iomgr/event_engine_shims/tcp_client.h +44 -0
  975. data/src/core/lib/iomgr/exec_ctx.cc +34 -56
  976. data/src/core/lib/iomgr/exec_ctx.h +152 -177
  977. data/src/core/lib/iomgr/executor.cc +21 -31
  978. data/src/core/lib/iomgr/executor.h +27 -30
  979. data/src/core/lib/iomgr/fork_posix.cc +30 -27
  980. data/src/core/lib/iomgr/fork_windows.cc +21 -21
  981. data/src/core/lib/iomgr/gethostname.h +20 -20
  982. data/src/core/lib/iomgr/gethostname_fallback.cc +17 -17
  983. data/src/core/lib/iomgr/gethostname_host_name_max.cc +17 -17
  984. data/src/core/lib/iomgr/gethostname_sysconf.cc +17 -17
  985. data/src/core/lib/iomgr/grpc_if_nametoindex.h +22 -22
  986. data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +20 -19
  987. data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +20 -19
  988. data/src/core/lib/iomgr/internal_errqueue.cc +41 -48
  989. data/src/core/lib/iomgr/internal_errqueue.h +84 -89
  990. data/src/core/lib/iomgr/iocp_windows.cc +23 -23
  991. data/src/core/lib/iomgr/iocp_windows.h +21 -21
  992. data/src/core/lib/iomgr/iomgr.cc +25 -20
  993. data/src/core/lib/iomgr/iomgr.h +35 -35
  994. data/src/core/lib/iomgr/iomgr_fwd.h +26 -0
  995. data/src/core/lib/iomgr/iomgr_internal.cc +17 -17
  996. data/src/core/lib/iomgr/iomgr_internal.h +28 -28
  997. data/src/core/lib/iomgr/iomgr_posix.cc +21 -20
  998. data/src/core/lib/iomgr/iomgr_posix_cfstream.cc +21 -19
  999. data/src/core/lib/iomgr/iomgr_windows.cc +24 -22
  1000. data/src/core/lib/iomgr/load_file.cc +24 -27
  1001. data/src/core/lib/iomgr/load_file.h +22 -22
  1002. data/src/core/lib/iomgr/lockfree_event.cc +114 -131
  1003. data/src/core/lib/iomgr/lockfree_event.h +23 -23
  1004. data/src/core/lib/iomgr/nameser.h +86 -86
  1005. data/src/core/lib/iomgr/polling_entity.cc +25 -21
  1006. data/src/core/lib/iomgr/polling_entity.h +29 -29
  1007. data/src/core/lib/iomgr/pollset.cc +17 -17
  1008. data/src/core/lib/iomgr/pollset.h +55 -55
  1009. data/src/core/lib/iomgr/pollset_set.cc +17 -17
  1010. data/src/core/lib/iomgr/pollset_set.h +26 -28
  1011. data/src/core/lib/iomgr/pollset_set_windows.cc +18 -18
  1012. data/src/core/lib/iomgr/pollset_set_windows.h +20 -20
  1013. data/src/core/lib/iomgr/pollset_windows.cc +32 -31
  1014. data/src/core/lib/iomgr/pollset_windows.h +24 -24
  1015. data/src/core/lib/iomgr/port.h +59 -37
  1016. data/src/core/lib/iomgr/python_util.h +24 -24
  1017. data/src/core/lib/iomgr/resolve_address.cc +34 -20
  1018. data/src/core/lib/iomgr/resolve_address.h +73 -43
  1019. data/src/core/lib/iomgr/resolve_address_impl.h +7 -7
  1020. data/src/core/lib/iomgr/resolve_address_posix.cc +81 -62
  1021. data/src/core/lib/iomgr/resolve_address_posix.h +26 -10
  1022. data/src/core/lib/iomgr/resolve_address_windows.cc +64 -38
  1023. data/src/core/lib/iomgr/resolve_address_windows.h +26 -10
  1024. data/src/core/lib/iomgr/resolved_address.h +3 -3
  1025. data/src/core/lib/iomgr/sockaddr.h +25 -26
  1026. data/src/core/lib/iomgr/sockaddr_posix.h +23 -21
  1027. data/src/core/lib/iomgr/sockaddr_utils_posix.cc +20 -18
  1028. data/src/core/lib/iomgr/sockaddr_windows.h +23 -21
  1029. data/src/core/lib/iomgr/socket_factory_posix.cc +18 -18
  1030. data/src/core/lib/iomgr/socket_factory_posix.h +32 -32
  1031. data/src/core/lib/iomgr/socket_mutator.cc +19 -18
  1032. data/src/core/lib/iomgr/socket_mutator.h +39 -39
  1033. data/src/core/lib/iomgr/socket_utils.h +27 -27
  1034. data/src/core/lib/iomgr/socket_utils_common_posix.cc +82 -101
  1035. data/src/core/lib/iomgr/socket_utils_linux.cc +18 -17
  1036. data/src/core/lib/iomgr/socket_utils_posix.cc +103 -19
  1037. data/src/core/lib/iomgr/socket_utils_posix.h +176 -84
  1038. data/src/core/lib/iomgr/socket_utils_windows.cc +20 -19
  1039. data/src/core/lib/iomgr/socket_windows.cc +37 -36
  1040. data/src/core/lib/iomgr/socket_windows.h +59 -61
  1041. data/src/core/lib/iomgr/systemd_utils.cc +116 -0
  1042. data/src/core/lib/iomgr/systemd_utils.h +33 -0
  1043. data/src/core/lib/iomgr/tcp_client.cc +28 -24
  1044. data/src/core/lib/iomgr/tcp_client.h +49 -38
  1045. data/src/core/lib/iomgr/tcp_client_cfstream.cc +46 -35
  1046. data/src/core/lib/iomgr/tcp_client_posix.cc +243 -105
  1047. data/src/core/lib/iomgr/tcp_client_posix.h +55 -52
  1048. data/src/core/lib/iomgr/tcp_client_windows.cc +73 -53
  1049. data/src/core/lib/iomgr/tcp_posix.cc +505 -362
  1050. data/src/core/lib/iomgr/tcp_posix.h +32 -30
  1051. data/src/core/lib/iomgr/tcp_server.cc +33 -24
  1052. data/src/core/lib/iomgr/tcp_server.h +78 -69
  1053. data/src/core/lib/iomgr/tcp_server_posix.cc +381 -133
  1054. data/src/core/lib/iomgr/tcp_server_utils_posix.h +68 -55
  1055. data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +120 -82
  1056. data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +46 -43
  1057. data/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc +19 -19
  1058. data/src/core/lib/iomgr/tcp_server_windows.cc +116 -118
  1059. data/src/core/lib/iomgr/tcp_windows.cc +116 -98
  1060. data/src/core/lib/iomgr/tcp_windows.h +34 -35
  1061. data/src/core/lib/iomgr/timer.cc +17 -17
  1062. data/src/core/lib/iomgr/timer.h +68 -68
  1063. data/src/core/lib/iomgr/timer_generic.cc +125 -134
  1064. data/src/core/lib/iomgr/timer_generic.h +21 -21
  1065. data/src/core/lib/iomgr/timer_heap.cc +25 -25
  1066. data/src/core/lib/iomgr/timer_heap.h +22 -22
  1067. data/src/core/lib/iomgr/timer_manager.cc +31 -31
  1068. data/src/core/lib/iomgr/timer_manager.h +27 -27
  1069. data/src/core/lib/iomgr/unix_sockets_posix.cc +20 -21
  1070. data/src/core/lib/iomgr/unix_sockets_posix.h +21 -21
  1071. data/src/core/lib/iomgr/unix_sockets_posix_noop.cc +19 -17
  1072. data/src/core/lib/iomgr/wakeup_fd_eventfd.cc +23 -23
  1073. data/src/core/lib/iomgr/wakeup_fd_nospecial.cc +23 -23
  1074. data/src/core/lib/iomgr/wakeup_fd_pipe.cc +29 -26
  1075. data/src/core/lib/iomgr/wakeup_fd_pipe.h +20 -20
  1076. data/src/core/lib/iomgr/wakeup_fd_posix.cc +33 -30
  1077. data/src/core/lib/iomgr/wakeup_fd_posix.h +52 -54
  1078. data/src/core/lib/json/json.h +23 -27
  1079. data/src/core/{ext/filters/http/message_compress/message_decompress_filter.h → lib/json/json_args.h} +13 -10
  1080. data/src/core/lib/json/json_channel_args.h +42 -0
  1081. data/src/core/lib/json/json_object_loader.cc +217 -0
  1082. data/src/core/lib/json/json_object_loader.h +634 -0
  1083. data/src/core/lib/json/json_reader.cc +123 -90
  1084. data/src/core/lib/json/json_util.cc +14 -34
  1085. data/src/core/lib/json/json_util.h +17 -8
  1086. data/src/core/lib/json/json_writer.cc +62 -57
  1087. data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.cc +30 -60
  1088. data/src/core/{ext/filters/client_channel → lib/load_balancing}/lb_policy.h +47 -62
  1089. data/src/core/lib/load_balancing/lb_policy_factory.h +49 -0
  1090. data/src/core/lib/load_balancing/lb_policy_registry.cc +142 -0
  1091. data/src/core/lib/load_balancing/lb_policy_registry.h +82 -0
  1092. data/src/core/{ext/filters/client_channel → lib/load_balancing}/subchannel_interface.h +57 -52
  1093. data/src/core/lib/matchers/matchers.cc +12 -7
  1094. data/src/core/lib/matchers/matchers.h +5 -3
  1095. data/src/core/lib/promise/activity.cc +23 -8
  1096. data/src/core/lib/promise/activity.h +116 -58
  1097. data/src/core/lib/promise/arena_promise.h +115 -72
  1098. data/src/core/lib/promise/context.h +18 -11
  1099. data/src/core/lib/promise/detail/basic_join.h +197 -0
  1100. data/src/core/lib/promise/detail/basic_seq.h +28 -33
  1101. data/src/core/lib/promise/detail/promise_factory.h +61 -14
  1102. data/src/core/lib/promise/detail/promise_like.h +3 -3
  1103. data/src/core/lib/promise/detail/status.h +31 -3
  1104. data/src/core/lib/promise/detail/switch.h +21 -21
  1105. data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h +23 -15
  1106. data/src/core/lib/promise/if.h +195 -0
  1107. data/src/core/lib/promise/interceptor_list.h +308 -0
  1108. data/src/core/lib/promise/intra_activity_waiter.h +9 -3
  1109. data/src/core/lib/promise/latch.h +109 -22
  1110. data/src/core/lib/promise/loop.h +13 -9
  1111. data/src/core/lib/promise/map.h +4 -6
  1112. data/src/core/lib/promise/pipe.h +608 -0
  1113. data/src/core/lib/promise/poll.h +129 -11
  1114. data/src/core/lib/promise/promise.h +5 -5
  1115. data/src/core/lib/promise/race.h +6 -10
  1116. data/src/core/lib/promise/seq.h +32 -14
  1117. data/src/core/lib/promise/sleep.cc +58 -42
  1118. data/src/core/lib/promise/sleep.h +44 -26
  1119. data/src/core/lib/promise/trace.cc +20 -0
  1120. data/src/core/lib/promise/trace.h +24 -0
  1121. data/src/core/lib/promise/try_join.h +82 -0
  1122. data/src/core/lib/promise/try_seq.h +41 -23
  1123. data/src/core/lib/resolver/resolver.cc +17 -59
  1124. data/src/core/lib/resolver/resolver.h +21 -18
  1125. data/src/core/lib/resolver/resolver_factory.h +11 -10
  1126. data/src/core/lib/resolver/resolver_registry.cc +17 -9
  1127. data/src/core/lib/resolver/resolver_registry.h +15 -5
  1128. data/src/core/lib/resolver/server_address.cc +46 -34
  1129. data/src/core/lib/resolver/server_address.h +36 -35
  1130. data/src/core/lib/resource_quota/api.cc +23 -2
  1131. data/src/core/lib/resource_quota/api.h +13 -4
  1132. data/src/core/lib/resource_quota/arena.cc +85 -27
  1133. data/src/core/lib/resource_quota/arena.h +197 -25
  1134. data/src/core/lib/resource_quota/memory_quota.cc +322 -101
  1135. data/src/core/lib/resource_quota/memory_quota.h +191 -56
  1136. data/src/core/lib/resource_quota/periodic_update.cc +78 -0
  1137. data/src/core/lib/resource_quota/periodic_update.h +71 -0
  1138. data/src/core/lib/resource_quota/resource_quota.h +12 -4
  1139. data/src/core/lib/resource_quota/thread_quota.cc +2 -0
  1140. data/src/core/lib/resource_quota/thread_quota.h +7 -3
  1141. data/src/core/lib/resource_quota/trace.h +3 -3
  1142. data/src/core/lib/security/authorization/authorization_engine.h +3 -3
  1143. data/src/core/lib/security/authorization/authorization_policy_provider.h +10 -3
  1144. data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +3 -0
  1145. data/src/core/lib/security/authorization/evaluate_args.cc +17 -8
  1146. data/src/core/lib/security/authorization/evaluate_args.h +9 -6
  1147. data/src/core/lib/security/authorization/grpc_authorization_engine.cc +5 -1
  1148. data/src/core/lib/security/authorization/grpc_authorization_engine.h +10 -3
  1149. data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +19 -3
  1150. data/src/core/lib/security/authorization/grpc_server_authz_filter.h +16 -5
  1151. data/src/core/lib/security/authorization/matchers.cc +44 -30
  1152. data/src/core/lib/security/authorization/matchers.h +10 -3
  1153. data/src/core/lib/security/authorization/rbac_policy.cc +6 -2
  1154. data/src/core/lib/security/authorization/rbac_policy.h +10 -3
  1155. data/src/core/{ext/xds → lib/security/certificate_provider}/certificate_provider_factory.h +9 -4
  1156. data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +60 -0
  1157. data/src/core/lib/security/certificate_provider/certificate_provider_registry.h +70 -0
  1158. data/src/core/lib/security/context/security_context.cc +27 -25
  1159. data/src/core/lib/security/context/security_context.h +56 -34
  1160. data/src/core/lib/security/credentials/alts/alts_credentials.cc +31 -24
  1161. data/src/core/lib/security/credentials/alts/alts_credentials.h +63 -59
  1162. data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +19 -19
  1163. data/src/core/lib/security/credentials/alts/check_gcp_environment.h +43 -43
  1164. data/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +17 -17
  1165. data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +18 -17
  1166. data/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc +18 -17
  1167. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +20 -22
  1168. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +18 -18
  1169. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h +38 -39
  1170. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc +19 -22
  1171. data/src/core/lib/security/credentials/call_creds_util.cc +10 -0
  1172. data/src/core/lib/security/credentials/call_creds_util.h +4 -3
  1173. data/src/core/lib/security/credentials/channel_creds_registry.h +10 -4
  1174. data/src/core/lib/security/credentials/channel_creds_registry_init.cc +13 -3
  1175. data/src/core/lib/security/credentials/composite/composite_credentials.cc +34 -34
  1176. data/src/core/lib/security/credentials/composite/composite_credentials.h +45 -33
  1177. data/src/core/lib/security/credentials/credentials.cc +23 -26
  1178. data/src/core/lib/security/credentials/credentials.h +79 -65
  1179. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +202 -74
  1180. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +21 -3
  1181. data/src/core/lib/security/credentials/external/aws_request_signer.cc +14 -3
  1182. data/src/core/lib/security/credentials/external/aws_request_signer.h +3 -3
  1183. data/src/core/lib/security/credentials/external/external_account_credentials.cc +86 -68
  1184. data/src/core/lib/security/credentials/external/external_account_credentials.h +14 -3
  1185. data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +29 -26
  1186. data/src/core/lib/security/credentials/external/file_external_account_credentials.h +9 -3
  1187. data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +44 -31
  1188. data/src/core/lib/security/credentials/external/url_external_account_credentials.h +13 -3
  1189. data/src/core/lib/security/credentials/fake/fake_credentials.cc +43 -39
  1190. data/src/core/lib/security/credentials/fake/fake_credentials.h +51 -44
  1191. data/src/core/lib/security/credentials/google_default/credentials_generic.cc +23 -25
  1192. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +146 -114
  1193. data/src/core/lib/security/credentials/google_default/google_default_credentials.h +33 -27
  1194. data/src/core/lib/security/credentials/iam/iam_credentials.cc +31 -21
  1195. data/src/core/lib/security/credentials/iam/iam_credentials.h +33 -23
  1196. data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +15 -5
  1197. data/src/core/lib/security/credentials/insecure/insecure_credentials.h +14 -10
  1198. data/src/core/lib/security/credentials/jwt/json_token.cc +40 -29
  1199. data/src/core/lib/security/credentials/jwt/json_token.h +36 -36
  1200. data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +32 -29
  1201. data/src/core/lib/security/credentials/jwt/jwt_credentials.h +37 -23
  1202. data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +113 -89
  1203. data/src/core/lib/security/credentials/jwt/jwt_verifier.h +49 -47
  1204. data/src/core/lib/security/credentials/local/local_credentials.cc +32 -26
  1205. data/src/core/lib/security/credentials/local/local_credentials.h +34 -28
  1206. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +95 -87
  1207. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +46 -25
  1208. data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +35 -31
  1209. data/src/core/lib/security/credentials/plugin/plugin_credentials.h +48 -24
  1210. data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +44 -42
  1211. data/src/core/lib/security/credentials/ssl/ssl_credentials.h +39 -27
  1212. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc +22 -30
  1213. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h +16 -17
  1214. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +65 -48
  1215. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +27 -16
  1216. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +36 -4
  1217. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +37 -14
  1218. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +22 -23
  1219. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +3 -3
  1220. data/src/core/lib/security/credentials/tls/tls_credentials.cc +46 -44
  1221. data/src/core/lib/security/credentials/tls/tls_credentials.h +30 -27
  1222. data/src/core/lib/security/credentials/tls/tls_utils.cc +5 -1
  1223. data/src/core/lib/security/credentials/tls/tls_utils.h +4 -4
  1224. data/src/core/lib/security/credentials/xds/xds_credentials.cc +33 -35
  1225. data/src/core/lib/security/credentials/xds/xds_credentials.h +24 -10
  1226. data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +60 -54
  1227. data/src/core/lib/security/security_connector/alts/alts_security_connector.h +49 -47
  1228. data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +75 -79
  1229. data/src/core/lib/security/security_connector/fake/fake_security_connector.h +25 -26
  1230. data/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc +15 -6
  1231. data/src/core/lib/security/security_connector/insecure/insecure_security_connector.h +26 -15
  1232. data/src/core/lib/security/security_connector/load_system_roots.h +20 -20
  1233. data/src/core/lib/security/security_connector/load_system_roots_fallback.cc +22 -20
  1234. data/src/core/lib/security/security_connector/{load_system_roots_linux.cc → load_system_roots_supported.cc} +43 -38
  1235. data/src/core/lib/security/security_connector/load_system_roots_supported.h +45 -0
  1236. data/src/core/lib/security/security_connector/local/local_security_connector.cc +64 -47
  1237. data/src/core/lib/security/security_connector/local/local_security_connector.h +49 -46
  1238. data/src/core/lib/security/security_connector/security_connector.cc +37 -35
  1239. data/src/core/lib/security/security_connector/security_connector.h +58 -45
  1240. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +55 -46
  1241. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +44 -43
  1242. data/src/core/lib/security/security_connector/ssl_utils.cc +47 -39
  1243. data/src/core/lib/security/security_connector/ssl_utils.h +45 -42
  1244. data/src/core/lib/security/security_connector/ssl_utils_config.cc +22 -22
  1245. data/src/core/lib/security/security_connector/ssl_utils_config.h +21 -22
  1246. data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +58 -56
  1247. data/src/core/lib/security/security_connector/tls/tls_security_connector.h +48 -28
  1248. data/src/core/lib/security/transport/auth_filters.h +53 -24
  1249. data/src/core/lib/security/transport/client_auth_filter.cc +57 -40
  1250. data/src/core/lib/security/transport/secure_endpoint.cc +130 -75
  1251. data/src/core/lib/security/transport/secure_endpoint.h +28 -27
  1252. data/src/core/lib/security/transport/security_handshaker.cc +144 -114
  1253. data/src/core/lib/security/transport/security_handshaker.h +28 -23
  1254. data/src/core/lib/security/transport/server_auth_filter.cc +157 -266
  1255. data/src/core/lib/security/transport/tsi_error.cc +23 -20
  1256. data/src/core/lib/security/transport/tsi_error.h +20 -20
  1257. data/src/core/lib/security/util/json_util.cc +24 -24
  1258. data/src/core/lib/security/util/json_util.h +21 -23
  1259. data/src/core/lib/service_config/service_config.h +19 -12
  1260. data/src/core/lib/service_config/service_config_call_data.h +15 -5
  1261. data/src/core/lib/service_config/service_config_impl.cc +111 -150
  1262. data/src/core/lib/service_config/service_config_impl.h +27 -27
  1263. data/src/core/lib/service_config/service_config_parser.cc +16 -28
  1264. data/src/core/lib/service_config/service_config_parser.h +18 -19
  1265. data/src/core/lib/slice/b64.cc +26 -26
  1266. data/src/core/lib/slice/b64.h +34 -32
  1267. data/src/core/lib/slice/percent_encoding.cc +24 -30
  1268. data/src/core/lib/slice/percent_encoding.h +28 -34
  1269. data/src/core/lib/slice/slice.cc +59 -46
  1270. data/src/core/lib/slice/slice.h +64 -20
  1271. data/src/core/lib/slice/slice_buffer.cc +136 -76
  1272. data/src/core/lib/slice/slice_buffer.h +168 -0
  1273. data/src/core/lib/slice/slice_internal.h +38 -45
  1274. data/src/core/lib/slice/slice_refcount.cc +3 -18
  1275. data/src/core/lib/slice/slice_refcount.h +53 -19
  1276. data/src/core/lib/slice/slice_string_helpers.cc +17 -33
  1277. data/src/core/lib/slice/slice_string_helpers.h +23 -30
  1278. data/src/core/lib/surface/api_trace.cc +17 -17
  1279. data/src/core/lib/surface/api_trace.h +25 -25
  1280. data/src/core/lib/surface/builtins.cc +7 -2
  1281. data/src/core/lib/surface/builtins.h +3 -3
  1282. data/src/core/lib/surface/byte_buffer.cc +26 -21
  1283. data/src/core/lib/surface/byte_buffer_reader.cc +24 -24
  1284. data/src/core/lib/surface/call.cc +2232 -483
  1285. data/src/core/lib/surface/call.h +135 -42
  1286. data/src/core/lib/surface/call_details.cc +22 -23
  1287. data/src/core/lib/surface/call_log_batch.cc +25 -18
  1288. data/src/core/lib/surface/call_test_only.h +34 -31
  1289. data/src/core/lib/surface/call_trace.cc +123 -0
  1290. data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_event_engine.cc → lib/surface/call_trace.h} +12 -10
  1291. data/src/core/lib/surface/channel.cc +217 -284
  1292. data/src/core/lib/surface/channel.h +137 -90
  1293. data/src/core/lib/surface/channel_init.cc +17 -17
  1294. data/src/core/lib/surface/channel_init.h +22 -20
  1295. data/src/core/lib/surface/channel_ping.cc +26 -20
  1296. data/src/core/lib/surface/channel_stack_type.cc +21 -19
  1297. data/src/core/lib/surface/channel_stack_type.h +22 -22
  1298. data/src/core/lib/surface/completion_queue.cc +200 -222
  1299. data/src/core/lib/surface/completion_queue.h +42 -40
  1300. data/src/core/lib/surface/completion_queue_factory.cc +34 -28
  1301. data/src/core/lib/surface/completion_queue_factory.h +21 -23
  1302. data/src/core/lib/surface/event_string.cc +19 -24
  1303. data/src/core/lib/surface/event_string.h +21 -21
  1304. data/src/core/lib/surface/init.cc +57 -114
  1305. data/src/core/lib/surface/init.h +20 -28
  1306. data/src/core/lib/surface/init_internally.cc +25 -0
  1307. data/src/core/lib/surface/init_internally.h +37 -0
  1308. data/src/core/lib/surface/lame_client.cc +87 -130
  1309. data/src/core/lib/surface/lame_client.h +62 -24
  1310. data/src/core/lib/surface/metadata_array.cc +18 -17
  1311. data/src/core/lib/surface/server.cc +406 -174
  1312. data/src/core/lib/surface/server.h +56 -24
  1313. data/src/core/lib/surface/validate_metadata.cc +64 -77
  1314. data/src/core/lib/surface/validate_metadata.h +33 -21
  1315. data/src/core/lib/surface/version.cc +21 -21
  1316. data/src/core/lib/transport/bdp_estimator.cc +19 -19
  1317. data/src/core/lib/transport/bdp_estimator.h +22 -24
  1318. data/src/core/lib/transport/connectivity_state.cc +24 -23
  1319. data/src/core/lib/transport/connectivity_state.h +24 -25
  1320. data/src/core/lib/transport/error_utils.cc +57 -79
  1321. data/src/core/lib/transport/error_utils.h +29 -25
  1322. data/src/core/lib/{channel → transport}/handshaker.cc +75 -58
  1323. data/src/core/lib/{channel → transport}/handshaker.h +59 -48
  1324. data/src/core/lib/transport/handshaker_factory.h +74 -0
  1325. data/src/core/lib/transport/handshaker_registry.cc +61 -0
  1326. data/src/core/lib/transport/handshaker_registry.h +69 -0
  1327. data/src/core/lib/transport/http2_errors.h +22 -22
  1328. data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.cc +78 -65
  1329. data/src/core/lib/transport/http_connect_handshaker.h +42 -0
  1330. data/src/core/lib/transport/metadata_batch.cc +305 -0
  1331. data/src/core/lib/transport/metadata_batch.h +405 -304
  1332. data/src/core/lib/transport/parsed_metadata.cc +2 -4
  1333. data/src/core/lib/transport/parsed_metadata.h +33 -14
  1334. data/src/core/lib/transport/pid_controller.cc +20 -20
  1335. data/src/core/lib/transport/pid_controller.h +27 -27
  1336. data/src/core/lib/transport/status_conversion.cc +23 -23
  1337. data/src/core/lib/transport/status_conversion.h +24 -24
  1338. data/src/core/lib/transport/tcp_connect_handshaker.cc +245 -0
  1339. data/src/core/lib/transport/tcp_connect_handshaker.h +39 -0
  1340. data/src/core/lib/transport/timeout_encoding.cc +24 -28
  1341. data/src/core/lib/transport/timeout_encoding.h +25 -21
  1342. data/src/core/lib/transport/transport.cc +97 -57
  1343. data/src/core/lib/transport/transport.h +243 -213
  1344. data/src/core/lib/transport/transport_fwd.h +20 -0
  1345. data/src/core/lib/transport/transport_impl.h +55 -45
  1346. data/src/core/lib/transport/transport_op_string.cc +40 -39
  1347. data/src/core/lib/uri/uri_parser.cc +12 -4
  1348. data/src/core/lib/uri/uri_parser.h +3 -5
  1349. data/src/core/plugin_registry/grpc_plugin_registry.cc +60 -69
  1350. data/src/core/plugin_registry/grpc_plugin_registry_extra.cc +20 -39
  1351. data/src/core/tsi/alts/crypt/aes_gcm.cc +20 -20
  1352. data/src/core/tsi/alts/crypt/gsec.cc +26 -26
  1353. data/src/core/tsi/alts/crypt/gsec.h +336 -336
  1354. data/src/core/tsi/alts/frame_protector/alts_counter.cc +23 -23
  1355. data/src/core/tsi/alts/frame_protector/alts_counter.h +68 -68
  1356. data/src/core/tsi/alts/frame_protector/alts_crypter.cc +19 -19
  1357. data/src/core/tsi/alts/frame_protector/alts_crypter.h +209 -209
  1358. data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +72 -71
  1359. data/src/core/tsi/alts/frame_protector/alts_frame_protector.h +40 -40
  1360. data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc +18 -18
  1361. data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h +82 -83
  1362. data/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc +22 -22
  1363. data/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc +22 -22
  1364. data/src/core/tsi/alts/frame_protector/frame_handler.cc +26 -25
  1365. data/src/core/tsi/alts/frame_protector/frame_handler.h +169 -169
  1366. data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +117 -97
  1367. data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +104 -104
  1368. data/src/core/tsi/alts/handshaker/alts_shared_resource.cc +18 -17
  1369. data/src/core/tsi/alts/handshaker/alts_shared_resource.h +43 -44
  1370. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +51 -43
  1371. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +60 -60
  1372. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h +24 -24
  1373. data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +21 -20
  1374. data/src/core/tsi/alts/handshaker/alts_tsi_utils.h +38 -38
  1375. data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +19 -19
  1376. data/src/core/tsi/alts/handshaker/transport_security_common_api.h +105 -104
  1377. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +45 -44
  1378. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h +40 -41
  1379. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +38 -36
  1380. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h +35 -36
  1381. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol.h +67 -68
  1382. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +32 -31
  1383. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.h +55 -56
  1384. data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc +54 -53
  1385. data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h +141 -142
  1386. data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +67 -58
  1387. data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h +43 -44
  1388. data/src/core/tsi/fake_transport_security.cc +146 -110
  1389. data/src/core/tsi/fake_transport_security.h +36 -30
  1390. data/src/core/tsi/local_transport_security.cc +43 -38
  1391. data/src/core/tsi/local_transport_security.h +33 -33
  1392. data/src/core/tsi/ssl/key_logging/ssl_key_logging.cc +9 -4
  1393. data/src/core/tsi/ssl/key_logging/ssl_key_logging.h +3 -3
  1394. data/src/core/tsi/ssl/session_cache/ssl_session.h +23 -21
  1395. data/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc +21 -19
  1396. data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +18 -17
  1397. data/src/core/tsi/ssl/session_cache/ssl_session_cache.h +31 -22
  1398. data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +27 -20
  1399. data/src/core/tsi/ssl_transport_security.cc +299 -370
  1400. data/src/core/tsi/ssl_transport_security.h +206 -203
  1401. data/src/core/tsi/ssl_transport_security_utils.cc +250 -0
  1402. data/src/core/tsi/ssl_transport_security_utils.h +147 -0
  1403. data/src/core/tsi/ssl_types.h +27 -27
  1404. data/src/core/tsi/transport_security.cc +44 -32
  1405. data/src/core/tsi/transport_security.h +49 -48
  1406. data/src/core/tsi/transport_security_grpc.cc +23 -22
  1407. data/src/core/tsi/transport_security_grpc.h +44 -41
  1408. data/src/core/tsi/transport_security_interface.h +344 -332
  1409. data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.clang +2 -0
  1410. data/src/ruby/ext/grpc/ext-export-truffleruby-with-ruby-abi-version.gcc +7 -0
  1411. data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.clang +2 -0
  1412. data/src/ruby/ext/grpc/ext-export-with-ruby-abi-version.gcc +7 -0
  1413. data/src/ruby/ext/grpc/ext-export.gcc +1 -1
  1414. data/src/ruby/ext/grpc/extconf.rb +98 -20
  1415. data/src/ruby/ext/grpc/rb_call.c +1 -0
  1416. data/src/ruby/ext/grpc/rb_channel.c +1 -0
  1417. data/src/ruby/ext/grpc/rb_channel_args.c +1 -0
  1418. data/src/ruby/ext/grpc/rb_compression_options.c +1 -1
  1419. data/src/ruby/ext/grpc/rb_grpc.c +1 -0
  1420. data/src/ruby/ext/grpc/rb_grpc_imports.generated.c +38 -38
  1421. data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +60 -60
  1422. data/src/ruby/ext/grpc/rb_loader.c +6 -2
  1423. data/src/ruby/lib/grpc/errors.rb +1 -1
  1424. data/src/ruby/lib/grpc/generic/bidi_call.rb +2 -0
  1425. data/src/ruby/lib/grpc/version.rb +1 -1
  1426. data/src/ruby/pb/generate_proto_ruby.sh +0 -6
  1427. data/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +4 -0
  1428. data/src/ruby/spec/channel_spec.rb +5 -43
  1429. data/src/ruby/spec/client_server_spec.rb +20 -8
  1430. data/src/ruby/spec/generic/active_call_spec.rb +12 -3
  1431. data/src/ruby/spec/generic/client_stub_spec.rb +23 -23
  1432. data/src/ruby/spec/generic/server_interceptors_spec.rb +1 -1
  1433. data/src/ruby/spec/user_agent_spec.rb +1 -1
  1434. data/third_party/abseil-cpp/absl/algorithm/container.h +57 -58
  1435. data/third_party/abseil-cpp/absl/base/attributes.h +88 -41
  1436. data/third_party/abseil-cpp/absl/base/casts.h +61 -68
  1437. data/third_party/abseil-cpp/absl/base/config.h +221 -39
  1438. data/third_party/abseil-cpp/absl/base/internal/cycleclock.cc +12 -42
  1439. data/third_party/abseil-cpp/absl/base/internal/cycleclock.h +52 -2
  1440. data/third_party/abseil-cpp/absl/base/internal/cycleclock_config.h +55 -0
  1441. data/third_party/abseil-cpp/absl/base/internal/direct_mmap.h +5 -4
  1442. data/third_party/abseil-cpp/absl/base/internal/endian.h +17 -62
  1443. data/third_party/abseil-cpp/absl/base/internal/fast_type_id.h +2 -0
  1444. data/third_party/abseil-cpp/absl/base/internal/inline_variable.h +3 -3
  1445. data/third_party/abseil-cpp/absl/base/internal/invoke.h +54 -0
  1446. data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.cc +2 -2
  1447. data/third_party/abseil-cpp/absl/base/internal/low_level_alloc.h +1 -1
  1448. data/third_party/abseil-cpp/absl/base/internal/prefetch.h +138 -0
  1449. data/third_party/abseil-cpp/absl/base/internal/raw_logging.cc +39 -28
  1450. data/third_party/abseil-cpp/absl/base/internal/raw_logging.h +36 -36
  1451. data/third_party/abseil-cpp/absl/base/internal/spinlock.cc +6 -3
  1452. data/third_party/abseil-cpp/absl/base/internal/spinlock.h +10 -6
  1453. data/third_party/abseil-cpp/absl/base/internal/spinlock_linux.inc +2 -5
  1454. data/third_party/abseil-cpp/absl/base/internal/spinlock_win32.inc +4 -1
  1455. data/third_party/abseil-cpp/absl/base/internal/strerror.cc +88 -0
  1456. data/third_party/abseil-cpp/absl/base/internal/strerror.h +39 -0
  1457. data/third_party/abseil-cpp/absl/base/internal/sysinfo.cc +14 -11
  1458. data/third_party/abseil-cpp/absl/base/internal/thread_annotations.h +9 -0
  1459. data/third_party/abseil-cpp/absl/base/internal/thread_identity.cc +2 -1
  1460. data/third_party/abseil-cpp/absl/base/internal/thread_identity.h +4 -0
  1461. data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc +6 -7
  1462. data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.h +12 -40
  1463. data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock_config.h +62 -0
  1464. data/third_party/abseil-cpp/absl/base/log_severity.cc +28 -0
  1465. data/third_party/abseil-cpp/absl/base/log_severity.h +51 -0
  1466. data/third_party/abseil-cpp/absl/base/macros.h +4 -21
  1467. data/third_party/abseil-cpp/absl/base/optimization.h +76 -16
  1468. data/third_party/abseil-cpp/absl/base/options.h +1 -7
  1469. data/third_party/abseil-cpp/absl/base/policy_checks.h +15 -13
  1470. data/third_party/abseil-cpp/absl/base/thread_annotations.h +2 -2
  1471. data/third_party/abseil-cpp/absl/cleanup/cleanup.h +140 -0
  1472. data/third_party/abseil-cpp/absl/cleanup/internal/cleanup.h +100 -0
  1473. data/third_party/abseil-cpp/absl/container/fixed_array.h +9 -5
  1474. data/third_party/abseil-cpp/absl/container/flat_hash_map.h +11 -4
  1475. data/third_party/abseil-cpp/absl/container/flat_hash_set.h +503 -0
  1476. data/third_party/abseil-cpp/absl/container/inlined_vector.h +84 -25
  1477. data/third_party/abseil-cpp/absl/container/internal/common.h +9 -8
  1478. data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h +132 -0
  1479. data/third_party/abseil-cpp/absl/container/internal/container_memory.h +23 -29
  1480. data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h +4 -55
  1481. data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc +116 -23
  1482. data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h +40 -54
  1483. data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h +165 -66
  1484. data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc +159 -4
  1485. data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h +1121 -470
  1486. data/third_party/abseil-cpp/absl/crc/crc32c.cc +99 -0
  1487. data/third_party/abseil-cpp/absl/crc/crc32c.h +183 -0
  1488. data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.cc +256 -0
  1489. data/third_party/abseil-cpp/absl/crc/internal/cpu_detect.h +57 -0
  1490. data/third_party/abseil-cpp/absl/crc/internal/crc.cc +468 -0
  1491. data/third_party/abseil-cpp/absl/crc/internal/crc.h +91 -0
  1492. data/third_party/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +269 -0
  1493. data/third_party/abseil-cpp/absl/crc/internal/crc32c.h +39 -0
  1494. data/third_party/abseil-cpp/absl/crc/internal/crc32c_inline.h +72 -0
  1495. data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc +130 -0
  1496. data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.h +159 -0
  1497. data/third_party/abseil-cpp/absl/crc/internal/crc_internal.h +179 -0
  1498. data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy.h +119 -0
  1499. data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +75 -0
  1500. data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_64.cc +434 -0
  1501. data/third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc +93 -0
  1502. data/third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +725 -0
  1503. data/third_party/abseil-cpp/absl/crc/internal/non_temporal_arm_intrinsics.h +79 -0
  1504. data/third_party/abseil-cpp/absl/crc/internal/non_temporal_memcpy.h +180 -0
  1505. data/third_party/abseil-cpp/absl/debugging/internal/address_is_readable.cc +45 -88
  1506. data/third_party/abseil-cpp/absl/debugging/internal/demangle.cc +67 -38
  1507. data/third_party/abseil-cpp/absl/debugging/internal/demangle.h +1 -1
  1508. data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc +16 -13
  1509. data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h +3 -2
  1510. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +19 -12
  1511. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_arm-inl.inc +8 -3
  1512. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_config.h +2 -1
  1513. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_generic-inl.inc +1 -1
  1514. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc +8 -3
  1515. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_riscv-inl.inc +59 -102
  1516. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_win32-inl.inc +5 -4
  1517. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_x86-inl.inc +41 -11
  1518. data/third_party/abseil-cpp/absl/debugging/internal/vdso_support.cc +18 -4
  1519. data/third_party/abseil-cpp/absl/debugging/symbolize.cc +6 -1
  1520. data/third_party/abseil-cpp/absl/debugging/symbolize_darwin.inc +3 -2
  1521. data/third_party/abseil-cpp/absl/debugging/symbolize_elf.inc +164 -101
  1522. data/third_party/abseil-cpp/absl/debugging/symbolize_win32.inc +7 -6
  1523. data/third_party/abseil-cpp/absl/functional/any_invocable.h +316 -0
  1524. data/third_party/abseil-cpp/absl/functional/bind_front.h +10 -1
  1525. data/third_party/abseil-cpp/absl/functional/function_ref.h +2 -1
  1526. data/third_party/abseil-cpp/absl/functional/internal/any_invocable.h +878 -0
  1527. data/third_party/abseil-cpp/absl/hash/hash.h +82 -8
  1528. data/third_party/abseil-cpp/absl/hash/internal/city.cc +10 -10
  1529. data/third_party/abseil-cpp/absl/hash/internal/hash.h +235 -26
  1530. data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc +3 -14
  1531. data/third_party/abseil-cpp/absl/memory/memory.h +26 -447
  1532. data/third_party/abseil-cpp/absl/meta/type_traits.h +104 -12
  1533. data/third_party/abseil-cpp/absl/numeric/bits.h +1 -1
  1534. data/third_party/abseil-cpp/absl/numeric/int128.cc +14 -10
  1535. data/third_party/abseil-cpp/absl/numeric/int128.h +2 -2
  1536. data/third_party/abseil-cpp/absl/profiling/internal/sample_recorder.h +33 -10
  1537. data/third_party/abseil-cpp/absl/random/bernoulli_distribution.h +4 -4
  1538. data/third_party/abseil-cpp/absl/random/distributions.h +3 -3
  1539. data/third_party/abseil-cpp/absl/random/internal/distribution_caller.h +3 -0
  1540. data/third_party/abseil-cpp/absl/random/internal/fast_uniform_bits.h +4 -2
  1541. data/third_party/abseil-cpp/absl/random/internal/generate_real.h +2 -2
  1542. data/third_party/abseil-cpp/absl/random/internal/nonsecure_base.h +59 -48
  1543. data/third_party/abseil-cpp/absl/random/internal/pcg_engine.h +3 -24
  1544. data/third_party/abseil-cpp/absl/random/internal/pool_urbg.cc +9 -9
  1545. data/third_party/abseil-cpp/absl/random/internal/randen.h +5 -11
  1546. data/third_party/abseil-cpp/absl/random/internal/randen_detect.cc +6 -2
  1547. data/third_party/abseil-cpp/absl/random/internal/randen_engine.h +48 -23
  1548. data/third_party/abseil-cpp/absl/random/internal/salted_seed_seq.h +24 -26
  1549. data/third_party/abseil-cpp/absl/random/internal/seed_material.cc +2 -2
  1550. data/third_party/abseil-cpp/absl/random/internal/traits.h +53 -5
  1551. data/third_party/abseil-cpp/absl/random/internal/uniform_helper.h +5 -5
  1552. data/third_party/abseil-cpp/absl/random/internal/wide_multiply.h +33 -48
  1553. data/third_party/abseil-cpp/absl/random/log_uniform_int_distribution.h +9 -10
  1554. data/third_party/abseil-cpp/absl/random/poisson_distribution.h +7 -4
  1555. data/third_party/abseil-cpp/absl/random/random.h +6 -6
  1556. data/third_party/abseil-cpp/absl/random/seed_sequences.h +1 -0
  1557. data/third_party/abseil-cpp/absl/random/uniform_int_distribution.h +2 -2
  1558. data/third_party/abseil-cpp/absl/random/uniform_real_distribution.h +1 -1
  1559. data/third_party/abseil-cpp/absl/random/zipf_distribution.h +4 -3
  1560. data/third_party/abseil-cpp/absl/status/internal/status_internal.h +18 -0
  1561. data/third_party/abseil-cpp/absl/status/status.cc +193 -14
  1562. data/third_party/abseil-cpp/absl/status/status.h +24 -14
  1563. data/third_party/abseil-cpp/absl/status/statusor.h +9 -3
  1564. data/third_party/abseil-cpp/absl/strings/ascii.cc +5 -5
  1565. data/third_party/abseil-cpp/absl/strings/ascii.h +4 -4
  1566. data/third_party/abseil-cpp/absl/strings/charconv.cc +534 -96
  1567. data/third_party/abseil-cpp/absl/strings/cord.cc +257 -924
  1568. data/third_party/abseil-cpp/absl/strings/cord.h +268 -156
  1569. data/third_party/abseil-cpp/absl/strings/cord_analysis.cc +188 -0
  1570. data/third_party/abseil-cpp/absl/strings/cord_analysis.h +44 -0
  1571. data/third_party/abseil-cpp/absl/strings/cord_buffer.cc +30 -0
  1572. data/third_party/abseil-cpp/absl/strings/cord_buffer.h +575 -0
  1573. data/third_party/abseil-cpp/absl/strings/escaping.cc +73 -62
  1574. data/third_party/abseil-cpp/absl/strings/escaping.h +24 -19
  1575. data/third_party/abseil-cpp/absl/strings/internal/char_map.h +14 -12
  1576. data/third_party/abseil-cpp/absl/strings/internal/charconv_bigint.cc +4 -4
  1577. data/third_party/abseil-cpp/absl/strings/internal/charconv_parse.cc +2 -2
  1578. data/third_party/abseil-cpp/absl/strings/internal/cord_data_edge.h +63 -0
  1579. data/third_party/abseil-cpp/absl/strings/internal/cord_internal.cc +20 -32
  1580. data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +441 -150
  1581. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.cc +157 -53
  1582. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h +70 -73
  1583. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.cc +8 -6
  1584. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_navigator.h +4 -2
  1585. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.cc +3 -2
  1586. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree_reader.h +5 -4
  1587. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_consume.cc +7 -74
  1588. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.cc +56 -0
  1589. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_crc.h +103 -0
  1590. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_flat.h +58 -17
  1591. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_ring.cc +13 -11
  1592. data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.h +7 -15
  1593. data/third_party/abseil-cpp/absl/strings/internal/cordz_handle.h +3 -3
  1594. data/third_party/abseil-cpp/absl/strings/internal/cordz_info.cc +19 -43
  1595. data/third_party/abseil-cpp/absl/strings/internal/cordz_info.h +7 -7
  1596. data/third_party/abseil-cpp/absl/strings/internal/cordz_statistics.h +5 -4
  1597. data/third_party/abseil-cpp/absl/strings/internal/cordz_update_tracker.h +4 -2
  1598. data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.cc +93 -0
  1599. data/third_party/abseil-cpp/absl/strings/internal/damerau_levenshtein_distance.h +34 -0
  1600. data/third_party/abseil-cpp/absl/strings/internal/escaping.cc +18 -15
  1601. data/third_party/abseil-cpp/absl/strings/internal/escaping.h +7 -9
  1602. data/third_party/abseil-cpp/absl/strings/internal/has_absl_stringify.h +55 -0
  1603. data/third_party/abseil-cpp/absl/strings/internal/memutil.cc +9 -6
  1604. data/third_party/abseil-cpp/absl/strings/internal/ostringstream.cc +14 -7
  1605. data/third_party/abseil-cpp/absl/strings/internal/ostringstream.h +35 -10
  1606. data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.cc +113 -46
  1607. data/third_party/abseil-cpp/absl/strings/internal/str_format/arg.h +127 -30
  1608. data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.cc +3 -2
  1609. data/third_party/abseil-cpp/absl/strings/internal/str_format/bind.h +42 -10
  1610. data/third_party/abseil-cpp/absl/strings/internal/str_format/checker.h +56 -289
  1611. data/third_party/abseil-cpp/absl/strings/internal/str_format/constexpr_parser.h +351 -0
  1612. data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.cc +6 -6
  1613. data/third_party/abseil-cpp/absl/strings/internal/str_format/extension.h +9 -4
  1614. data/third_party/abseil-cpp/absl/strings/internal/str_format/float_conversion.cc +215 -181
  1615. data/third_party/abseil-cpp/absl/strings/internal/str_format/output.h +2 -1
  1616. data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.cc +10 -209
  1617. data/third_party/abseil-cpp/absl/strings/internal/str_format/parser.h +14 -103
  1618. data/third_party/abseil-cpp/absl/strings/internal/str_join_internal.h +9 -6
  1619. data/third_party/abseil-cpp/absl/strings/internal/str_split_internal.h +2 -1
  1620. data/third_party/abseil-cpp/absl/strings/internal/string_constant.h +10 -2
  1621. data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.cc +28 -0
  1622. data/third_party/abseil-cpp/absl/strings/internal/stringify_sink.h +57 -0
  1623. data/third_party/abseil-cpp/absl/strings/internal/utf8.cc +9 -9
  1624. data/third_party/abseil-cpp/absl/strings/numbers.cc +42 -39
  1625. data/third_party/abseil-cpp/absl/strings/numbers.h +26 -23
  1626. data/third_party/abseil-cpp/absl/strings/str_cat.cc +9 -6
  1627. data/third_party/abseil-cpp/absl/strings/str_cat.h +70 -16
  1628. data/third_party/abseil-cpp/absl/strings/str_format.h +71 -9
  1629. data/third_party/abseil-cpp/absl/strings/str_join.h +9 -15
  1630. data/third_party/abseil-cpp/absl/strings/str_split.h +1 -2
  1631. data/third_party/abseil-cpp/absl/strings/string_view.cc +8 -19
  1632. data/third_party/abseil-cpp/absl/strings/string_view.h +6 -12
  1633. data/third_party/abseil-cpp/absl/strings/strip.h +8 -6
  1634. data/third_party/abseil-cpp/absl/strings/substitute.cc +8 -6
  1635. data/third_party/abseil-cpp/absl/strings/substitute.h +55 -21
  1636. data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.cc +9 -6
  1637. data/third_party/abseil-cpp/absl/synchronization/internal/create_thread_identity.h +0 -4
  1638. data/third_party/abseil-cpp/absl/synchronization/internal/futex.h +20 -17
  1639. data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc +37 -31
  1640. data/third_party/abseil-cpp/absl/synchronization/internal/kernel_timeout.h +22 -8
  1641. data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.cc +0 -4
  1642. data/third_party/abseil-cpp/absl/synchronization/internal/per_thread_sem.h +1 -6
  1643. data/third_party/abseil-cpp/absl/synchronization/internal/waiter.cc +0 -25
  1644. data/third_party/abseil-cpp/absl/synchronization/internal/waiter.h +10 -4
  1645. data/third_party/abseil-cpp/absl/synchronization/mutex.cc +172 -88
  1646. data/third_party/abseil-cpp/absl/synchronization/mutex.h +102 -55
  1647. data/third_party/abseil-cpp/absl/synchronization/notification.cc +0 -1
  1648. data/third_party/abseil-cpp/absl/synchronization/notification.h +3 -3
  1649. data/third_party/abseil-cpp/absl/time/civil_time.cc +26 -0
  1650. data/third_party/abseil-cpp/absl/time/civil_time.h +25 -0
  1651. data/third_party/abseil-cpp/absl/time/clock.cc +17 -11
  1652. data/third_party/abseil-cpp/absl/time/duration.cc +12 -11
  1653. data/third_party/abseil-cpp/absl/time/format.cc +2 -1
  1654. data/third_party/abseil-cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h +11 -7
  1655. data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_format.cc +1 -1
  1656. data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_info.cc +26 -5
  1657. data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc +7 -6
  1658. data/third_party/abseil-cpp/absl/time/internal/cctz/src/zone_info_source.cc +36 -35
  1659. data/third_party/abseil-cpp/absl/time/time.cc +2 -2
  1660. data/third_party/abseil-cpp/absl/time/time.h +268 -169
  1661. data/third_party/abseil-cpp/absl/types/internal/optional.h +8 -0
  1662. data/third_party/abseil-cpp/absl/types/internal/span.h +30 -19
  1663. data/third_party/abseil-cpp/absl/types/internal/variant.h +28 -40
  1664. data/third_party/abseil-cpp/absl/types/optional.h +17 -14
  1665. data/third_party/abseil-cpp/absl/types/span.h +31 -8
  1666. data/third_party/boringssl-with-bazel/src/crypto/x509v3/v3_genn.c +1 -1
  1667. data/third_party/boringssl-with-bazel/src/include/openssl/x509v3.h +1 -2
  1668. data/third_party/re2/re2/bitstate.cc +3 -3
  1669. data/third_party/re2/re2/dfa.cc +13 -13
  1670. data/third_party/re2/re2/nfa.cc +4 -4
  1671. data/third_party/re2/re2/onepass.cc +2 -2
  1672. data/third_party/re2/re2/prefilter_tree.cc +27 -59
  1673. data/third_party/re2/re2/prefilter_tree.h +3 -2
  1674. data/third_party/re2/re2/prog.cc +11 -2
  1675. data/third_party/re2/re2/prog.h +17 -5
  1676. data/third_party/re2/re2/re2.cc +6 -11
  1677. data/third_party/re2/re2/re2.h +1 -1
  1678. data/third_party/re2/re2/regexp.cc +1 -2
  1679. data/third_party/re2/re2/stringpiece.h +10 -7
  1680. data/third_party/re2/re2/unicode_casefold.cc +25 -11
  1681. data/third_party/re2/re2/unicode_groups.cc +319 -151
  1682. data/third_party/re2/re2/walker-inl.h +3 -2
  1683. data/third_party/re2/util/mutex.h +4 -4
  1684. data/third_party/upb/third_party/utf8_range/utf8_range.h +1 -1
  1685. data/third_party/upb/upb/arena.c +277 -0
  1686. data/third_party/upb/upb/arena.h +225 -0
  1687. data/third_party/upb/upb/array.c +114 -0
  1688. data/third_party/upb/upb/array.h +83 -0
  1689. data/third_party/upb/upb/collections.h +36 -0
  1690. data/third_party/upb/upb/decode.c +161 -65
  1691. data/third_party/upb/upb/decode.h +1 -0
  1692. data/third_party/upb/upb/decode_fast.c +1 -1
  1693. data/third_party/upb/upb/def.c +10 -2
  1694. data/third_party/upb/upb/def.h +8 -1
  1695. data/third_party/upb/upb/def.hpp +7 -4
  1696. data/third_party/upb/upb/encode.c +29 -20
  1697. data/third_party/upb/upb/encode.h +16 -6
  1698. data/third_party/upb/upb/extension_registry.c +93 -0
  1699. data/third_party/upb/upb/extension_registry.h +84 -0
  1700. data/third_party/upb/upb/{decode_internal.h → internal/decode.h} +5 -5
  1701. data/third_party/upb/upb/internal/table.h +385 -0
  1702. data/third_party/upb/upb/{upb_internal.h → internal/upb.h} +3 -3
  1703. data/third_party/upb/upb/internal/vsnprintf_compat.h +52 -0
  1704. data/third_party/upb/upb/json_decode.c +1512 -0
  1705. data/third_party/upb/upb/json_decode.h +47 -0
  1706. data/third_party/upb/upb/json_encode.c +7 -3
  1707. data/third_party/upb/upb/json_encode.h +6 -3
  1708. data/third_party/upb/upb/map.c +108 -0
  1709. data/third_party/upb/upb/map.h +117 -0
  1710. data/third_party/upb/upb/message_value.h +66 -0
  1711. data/third_party/upb/upb/mini_table.c +1147 -0
  1712. data/third_party/upb/upb/mini_table.h +189 -0
  1713. data/third_party/upb/upb/mini_table.hpp +112 -0
  1714. data/third_party/upb/upb/msg.c +2 -62
  1715. data/third_party/upb/upb/msg.h +2 -45
  1716. data/third_party/upb/upb/msg_internal.h +28 -22
  1717. data/third_party/upb/upb/port_def.inc +2 -1
  1718. data/third_party/upb/upb/port_undef.inc +1 -0
  1719. data/third_party/upb/upb/reflection.c +2 -159
  1720. data/third_party/upb/upb/reflection.h +2 -112
  1721. data/third_party/upb/upb/status.c +86 -0
  1722. data/third_party/upb/upb/status.h +66 -0
  1723. data/third_party/upb/upb/table.c +2 -2
  1724. data/third_party/upb/upb/table_internal.h +3 -352
  1725. data/third_party/upb/upb/text_encode.c +3 -2
  1726. data/third_party/upb/upb/upb.c +4 -290
  1727. data/third_party/upb/upb/upb.h +7 -196
  1728. data/third_party/zlib/compress.c +3 -3
  1729. data/third_party/zlib/crc32.c +21 -12
  1730. data/third_party/zlib/deflate.c +112 -106
  1731. data/third_party/zlib/deflate.h +2 -2
  1732. data/third_party/zlib/gzlib.c +1 -1
  1733. data/third_party/zlib/gzread.c +3 -5
  1734. data/third_party/zlib/gzwrite.c +1 -1
  1735. data/third_party/zlib/infback.c +10 -7
  1736. data/third_party/zlib/inflate.c +5 -2
  1737. data/third_party/zlib/inftrees.c +2 -2
  1738. data/third_party/zlib/inftrees.h +1 -1
  1739. data/third_party/zlib/trees.c +61 -62
  1740. data/third_party/zlib/uncompr.c +2 -2
  1741. data/third_party/zlib/zconf.h +16 -3
  1742. data/third_party/zlib/zlib.h +10 -10
  1743. data/third_party/zlib/zutil.c +9 -7
  1744. data/third_party/zlib/zutil.h +1 -0
  1745. metadata +372 -102
  1746. data/include/grpc/impl/codegen/gpr_slice.h +0 -71
  1747. data/src/core/ext/filters/client_channel/http_connect_handshaker.h +0 -42
  1748. data/src/core/ext/filters/client_channel/lb_policy_factory.h +0 -48
  1749. data/src/core/ext/filters/client_channel/lb_policy_registry.cc +0 -185
  1750. data/src/core/ext/filters/client_channel/lb_policy_registry.h +0 -65
  1751. data/src/core/ext/filters/client_channel/proxy_mapper.h +0 -54
  1752. data/src/core/ext/filters/client_channel/proxy_mapper_registry.cc +0 -89
  1753. data/src/core/ext/filters/client_channel/proxy_mapper_registry.h +0 -50
  1754. data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +0 -186
  1755. data/src/core/ext/filters/fault_injection/service_config_parser.cc +0 -179
  1756. data/src/core/ext/filters/http/message_compress/message_compress_filter.cc +0 -456
  1757. data/src/core/ext/filters/http/message_compress/message_compress_filter.h +0 -53
  1758. data/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +0 -386
  1759. data/src/core/ext/filters/server_config_selector/server_config_selector.cc +0 -61
  1760. data/src/core/ext/xds/certificate_provider_registry.cc +0 -103
  1761. data/src/core/ext/xds/certificate_provider_registry.h +0 -57
  1762. data/src/core/lib/channel/handshaker_factory.h +0 -50
  1763. data/src/core/lib/channel/handshaker_registry.cc +0 -50
  1764. data/src/core/lib/channel/handshaker_registry.h +0 -71
  1765. data/src/core/lib/event_engine/sockaddr.cc +0 -40
  1766. data/src/core/lib/event_engine/sockaddr.h +0 -44
  1767. data/src/core/lib/gpr/env.h +0 -40
  1768. data/src/core/lib/gpr/env_linux.cc +0 -75
  1769. data/src/core/lib/gpr/env_posix.cc +0 -46
  1770. data/src/core/lib/gpr/env_windows.cc +0 -74
  1771. data/src/core/lib/gpr/murmur_hash.cc +0 -82
  1772. data/src/core/lib/gpr/murmur_hash.h +0 -29
  1773. data/src/core/lib/gpr/string_windows.h +0 -32
  1774. data/src/core/lib/gpr/tls.h +0 -158
  1775. data/src/core/lib/gprpp/capture.h +0 -76
  1776. data/src/core/lib/iomgr/error_internal.h +0 -66
  1777. data/src/core/lib/iomgr/event_engine/endpoint.cc +0 -172
  1778. data/src/core/lib/iomgr/event_engine/endpoint.h +0 -52
  1779. data/src/core/lib/iomgr/event_engine/iomgr.cc +0 -85
  1780. data/src/core/lib/iomgr/event_engine/pollset.cc +0 -87
  1781. data/src/core/lib/iomgr/event_engine/resolved_address_internal.cc +0 -47
  1782. data/src/core/lib/iomgr/event_engine/resolver.cc +0 -133
  1783. data/src/core/lib/iomgr/event_engine/resolver.h +0 -56
  1784. data/src/core/lib/iomgr/event_engine/tcp.cc +0 -296
  1785. data/src/core/lib/iomgr/event_engine/timer.cc +0 -62
  1786. data/src/core/lib/iomgr/executor/mpmcqueue.cc +0 -182
  1787. data/src/core/lib/iomgr/executor/mpmcqueue.h +0 -171
  1788. data/src/core/lib/iomgr/executor/threadpool.cc +0 -136
  1789. data/src/core/lib/iomgr/executor/threadpool.h +0 -150
  1790. data/src/core/lib/iomgr/time_averaged_stats.cc +0 -64
  1791. data/src/core/lib/iomgr/time_averaged_stats.h +0 -72
  1792. data/src/core/lib/profiling/basic_timers.cc +0 -295
  1793. data/src/core/lib/profiling/stap_timers.cc +0 -50
  1794. data/src/core/lib/profiling/timers.h +0 -94
  1795. data/src/core/lib/promise/call_push_pull.h +0 -144
  1796. data/src/core/lib/security/security_connector/load_system_roots_linux.h +0 -46
  1797. data/src/core/lib/slice/slice_api.cc +0 -39
  1798. data/src/core/lib/slice/slice_refcount_base.h +0 -61
  1799. data/src/core/lib/slice/slice_split.cc +0 -100
  1800. data/src/core/lib/slice/slice_split.h +0 -40
  1801. data/src/core/lib/transport/byte_stream.cc +0 -164
  1802. data/src/core/lib/transport/byte_stream.h +0 -166
  1803. data/third_party/abseil-cpp/absl/container/internal/have_sse.h +0 -50
@@ -1,60 +1,95 @@
1
- /*
2
- *
3
- * Copyright 2015 gRPC authors.
4
- *
5
- * Licensed under the Apache License, Version 2.0 (the "License");
6
- * you may not use this file except in compliance with the License.
7
- * You may obtain a copy of the License at
8
- *
9
- * http://www.apache.org/licenses/LICENSE-2.0
10
- *
11
- * Unless required by applicable law or agreed to in writing, software
12
- * distributed under the License is distributed on an "AS IS" BASIS,
13
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- * See the License for the specific language governing permissions and
15
- * limitations under the License.
16
- *
17
- */
1
+ //
2
+ //
3
+ // Copyright 2015 gRPC authors.
4
+ //
5
+ // Licensed under the Apache License, Version 2.0 (the "License");
6
+ // you may not use this file except in compliance with the License.
7
+ // You may obtain a copy of the License at
8
+ //
9
+ // http://www.apache.org/licenses/LICENSE-2.0
10
+ //
11
+ // Unless required by applicable law or agreed to in writing, software
12
+ // distributed under the License is distributed on an "AS IS" BASIS,
13
+ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ // See the License for the specific language governing permissions and
15
+ // limitations under the License.
16
+ //
17
+ //
18
18
 
19
19
  #include <grpc/support/port_platform.h>
20
20
 
21
21
  #include "src/core/lib/surface/call.h"
22
22
 
23
- #include <assert.h>
23
+ #include <inttypes.h>
24
24
  #include <limits.h>
25
- #include <stdio.h>
26
25
  #include <stdlib.h>
27
- #include <string.h>
28
26
 
27
+ #include <algorithm>
29
28
  #include <atomic>
29
+ #include <initializer_list>
30
+ #include <memory>
31
+ #include <new>
30
32
  #include <string>
31
-
33
+ #include <type_traits>
34
+ #include <utility>
35
+ #include <vector>
36
+
37
+ #include "absl/base/thread_annotations.h"
38
+ #include "absl/cleanup/cleanup.h"
39
+ #include "absl/meta/type_traits.h"
40
+ #include "absl/status/status.h"
32
41
  #include "absl/strings/str_cat.h"
33
42
  #include "absl/strings/str_format.h"
43
+ #include "absl/strings/str_join.h"
44
+ #include "absl/strings/string_view.h"
45
+ #include "absl/types/variant.h"
34
46
 
47
+ #include <grpc/byte_buffer.h>
35
48
  #include <grpc/compression.h>
49
+ #include <grpc/event_engine/event_engine.h>
36
50
  #include <grpc/grpc.h>
51
+ #include <grpc/impl/propagation_bits.h>
37
52
  #include <grpc/slice.h>
53
+ #include <grpc/slice_buffer.h>
54
+ #include <grpc/status.h>
38
55
  #include <grpc/support/alloc.h>
56
+ #include <grpc/support/atm.h>
39
57
  #include <grpc/support/log.h>
40
58
  #include <grpc/support/string_util.h>
59
+ #include <grpc/support/time.h>
41
60
 
61
+ #include "src/core/lib/channel/call_finalization.h"
42
62
  #include "src/core/lib/channel/channel_stack.h"
63
+ #include "src/core/lib/channel/channelz.h"
43
64
  #include "src/core/lib/channel/context.h"
65
+ #include "src/core/lib/channel/status_util.h"
44
66
  #include "src/core/lib/compression/compression_internal.h"
45
67
  #include "src/core/lib/debug/stats.h"
68
+ #include "src/core/lib/debug/stats_data.h"
69
+ #include "src/core/lib/experiments/experiments.h"
46
70
  #include "src/core/lib/gpr/alloc.h"
47
- #include "src/core/lib/gpr/string.h"
48
71
  #include "src/core/lib/gpr/time_precise.h"
49
72
  #include "src/core/lib/gpr/useful.h"
50
- #include "src/core/lib/gprpp/manual_constructor.h"
73
+ #include "src/core/lib/gprpp/bitset.h"
74
+ #include "src/core/lib/gprpp/cpp_impl_of.h"
75
+ #include "src/core/lib/gprpp/crash.h"
76
+ #include "src/core/lib/gprpp/debug_location.h"
51
77
  #include "src/core/lib/gprpp/ref_counted.h"
52
- #include "src/core/lib/iomgr/timer.h"
53
- #include "src/core/lib/profiling/timers.h"
78
+ #include "src/core/lib/gprpp/ref_counted_ptr.h"
79
+ #include "src/core/lib/gprpp/status_helper.h"
80
+ #include "src/core/lib/gprpp/sync.h"
81
+ #include "src/core/lib/iomgr/call_combiner.h"
82
+ #include "src/core/lib/iomgr/exec_ctx.h"
83
+ #include "src/core/lib/iomgr/polling_entity.h"
84
+ #include "src/core/lib/promise/activity.h"
85
+ #include "src/core/lib/promise/arena_promise.h"
86
+ #include "src/core/lib/promise/context.h"
87
+ #include "src/core/lib/promise/detail/basic_seq.h"
88
+ #include "src/core/lib/promise/pipe.h"
89
+ #include "src/core/lib/promise/poll.h"
54
90
  #include "src/core/lib/resource_quota/arena.h"
91
+ #include "src/core/lib/slice/slice_buffer.h"
55
92
  #include "src/core/lib/slice/slice_internal.h"
56
- #include "src/core/lib/slice/slice_split.h"
57
- #include "src/core/lib/slice/slice_string_helpers.h"
58
93
  #include "src/core/lib/surface/api_trace.h"
59
94
  #include "src/core/lib/surface/call_test_only.h"
60
95
  #include "src/core/lib/surface/channel.h"
@@ -62,13 +97,19 @@
62
97
  #include "src/core/lib/surface/server.h"
63
98
  #include "src/core/lib/surface/validate_metadata.h"
64
99
  #include "src/core/lib/transport/error_utils.h"
100
+ #include "src/core/lib/transport/metadata_batch.h"
65
101
  #include "src/core/lib/transport/transport.h"
66
102
 
67
103
  grpc_core::TraceFlag grpc_call_error_trace(false, "call_error");
68
104
  grpc_core::TraceFlag grpc_compression_trace(false, "compression");
105
+ grpc_core::TraceFlag grpc_call_trace(false, "call");
106
+ grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount");
69
107
 
70
108
  namespace grpc_core {
71
109
 
110
+ ///////////////////////////////////////////////////////////////////////////////
111
+ // Call
112
+
72
113
  class Call : public CppImplOf<Call, grpc_call> {
73
114
  public:
74
115
  Arena* arena() { return arena_; }
@@ -81,12 +122,13 @@ class Call : public CppImplOf<Call, grpc_call> {
81
122
  void CancelWithStatus(grpc_status_code status, const char* description);
82
123
  virtual void CancelWithError(grpc_error_handle error) = 0;
83
124
  virtual void SetCompletionQueue(grpc_completion_queue* cq) = 0;
84
- virtual char* GetPeer() = 0;
125
+ char* GetPeer();
85
126
  virtual grpc_call_error StartBatch(const grpc_op* ops, size_t nops,
86
127
  void* notify_tag,
87
128
  bool is_notify_tag_closure) = 0;
88
129
  virtual bool failed_before_recv_message() const = 0;
89
130
  virtual bool is_trailers_only() const = 0;
131
+ virtual absl::string_view GetServerAuthority() const = 0;
90
132
  virtual void ExternalRef() = 0;
91
133
  virtual void ExternalUnref() = 0;
92
134
  virtual void InternalRef(const char* reason) = 0;
@@ -102,12 +144,19 @@ class Call : public CppImplOf<Call, grpc_call> {
102
144
  // for that functionality be invented)
103
145
  virtual grpc_call_stack* call_stack() = 0;
104
146
 
147
+ gpr_atm* peer_string_atm_ptr() { return &peer_string_; }
148
+
105
149
  protected:
106
- Call(Arena* arena, bool is_client, Timestamp send_deadline)
107
- : arena_(arena), send_deadline_(send_deadline), is_client_(is_client) {
108
- GPR_DEBUG_ASSERT(arena_ != nullptr);
109
- }
110
- ~Call() = default;
150
+ // The maximum number of concurrent batches possible.
151
+ // Based upon the maximum number of individually queueable ops in the batch
152
+ // api:
153
+ // - initial metadata send
154
+ // - message send
155
+ // - status/close send (depending on client/server)
156
+ // - initial metadata recv
157
+ // - message recv
158
+ // - status/close recv (depending on client/server)
159
+ static constexpr size_t kMaxConcurrentBatches = 6;
111
160
 
112
161
  struct ParentCall {
113
162
  Mutex child_list_mu;
@@ -117,15 +166,32 @@ class Call : public CppImplOf<Call, grpc_call> {
117
166
  struct ChildCall {
118
167
  explicit ChildCall(Call* parent) : parent(parent) {}
119
168
  Call* parent;
120
- /** siblings: children of the same parent form a list, and this list is
121
- protected under
122
- parent->mu */
169
+ /// siblings: children of the same parent form a list, and this list is
170
+ /// protected under
171
+ /// parent->mu
123
172
  Call* sibling_next = nullptr;
124
173
  Call* sibling_prev = nullptr;
125
174
  };
126
175
 
176
+ Call(Arena* arena, bool is_client, Timestamp send_deadline,
177
+ RefCountedPtr<Channel> channel)
178
+ : channel_(std::move(channel)),
179
+ arena_(arena),
180
+ send_deadline_(send_deadline),
181
+ is_client_(is_client) {
182
+ GPR_DEBUG_ASSERT(arena_ != nullptr);
183
+ GPR_DEBUG_ASSERT(channel_ != nullptr);
184
+ }
185
+ virtual ~Call() = default;
186
+
187
+ void DeleteThis();
188
+
127
189
  ParentCall* GetOrCreateParentCall();
128
190
  ParentCall* parent_call();
191
+ Channel* channel() {
192
+ GPR_DEBUG_ASSERT(channel_ != nullptr);
193
+ return channel_.get();
194
+ }
129
195
 
130
196
  absl::Status InitParent(Call* parent, uint32_t propagation_mask);
131
197
  void PublishToParent(Call* parent);
@@ -137,7 +203,10 @@ class Call : public CppImplOf<Call, grpc_call> {
137
203
  send_deadline_ = send_deadline;
138
204
  }
139
205
 
206
+ void ClearPeerString() { gpr_atm_rel_store(&peer_string_, 0); }
207
+
140
208
  private:
209
+ RefCountedPtr<Channel> channel_;
141
210
  Arena* const arena_;
142
211
  std::atomic<ParentCall*> parent_call_{nullptr};
143
212
  ChildCall* child_ = nullptr;
@@ -145,11 +214,151 @@ class Call : public CppImplOf<Call, grpc_call> {
145
214
  const bool is_client_;
146
215
  // flag indicating that cancellation is inherited
147
216
  bool cancellation_is_inherited_ = false;
217
+ // A char* indicating the peer name.
218
+ gpr_atm peer_string_ = 0;
148
219
  };
149
220
 
221
+ Call::ParentCall* Call::GetOrCreateParentCall() {
222
+ ParentCall* p = parent_call_.load(std::memory_order_acquire);
223
+ if (p == nullptr) {
224
+ p = arena_->New<ParentCall>();
225
+ ParentCall* expected = nullptr;
226
+ if (!parent_call_.compare_exchange_strong(expected, p,
227
+ std::memory_order_release,
228
+ std::memory_order_relaxed)) {
229
+ p->~ParentCall();
230
+ p = expected;
231
+ }
232
+ }
233
+ return p;
234
+ }
235
+
236
+ Call::ParentCall* Call::parent_call() {
237
+ return parent_call_.load(std::memory_order_acquire);
238
+ }
239
+
240
+ absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
241
+ child_ = arena()->New<ChildCall>(parent);
242
+
243
+ parent->InternalRef("child");
244
+ GPR_ASSERT(is_client_);
245
+ GPR_ASSERT(!parent->is_client_);
246
+
247
+ if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
248
+ send_deadline_ = std::min(send_deadline_, parent->send_deadline_);
249
+ }
250
+ // for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
251
+ // GRPC_PROPAGATE_STATS_CONTEXT
252
+ // TODO(ctiller): This should change to use the appropriate census start_op
253
+ // call.
254
+ if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
255
+ if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
256
+ return absl::UnknownError(
257
+ "Census tracing propagation requested without Census context "
258
+ "propagation");
259
+ }
260
+ ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
261
+ nullptr);
262
+ } else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
263
+ return absl::UnknownError(
264
+ "Census context propagation requested without Census tracing "
265
+ "propagation");
266
+ }
267
+ if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
268
+ cancellation_is_inherited_ = true;
269
+ }
270
+ return absl::OkStatus();
271
+ }
272
+
273
+ void Call::PublishToParent(Call* parent) {
274
+ ChildCall* cc = child_;
275
+ ParentCall* pc = parent->GetOrCreateParentCall();
276
+ MutexLock lock(&pc->child_list_mu);
277
+ if (pc->first_child == nullptr) {
278
+ pc->first_child = this;
279
+ cc->sibling_next = cc->sibling_prev = this;
280
+ } else {
281
+ cc->sibling_next = pc->first_child;
282
+ cc->sibling_prev = pc->first_child->child_->sibling_prev;
283
+ cc->sibling_next->child_->sibling_prev =
284
+ cc->sibling_prev->child_->sibling_next = this;
285
+ }
286
+ if (parent->Completed()) {
287
+ CancelWithError(absl::CancelledError());
288
+ }
289
+ }
290
+
291
+ void Call::MaybeUnpublishFromParent() {
292
+ ChildCall* cc = child_;
293
+ if (cc == nullptr) return;
294
+
295
+ ParentCall* pc = cc->parent->parent_call();
296
+ {
297
+ MutexLock lock(&pc->child_list_mu);
298
+ if (this == pc->first_child) {
299
+ pc->first_child = cc->sibling_next;
300
+ if (this == pc->first_child) {
301
+ pc->first_child = nullptr;
302
+ }
303
+ }
304
+ cc->sibling_prev->child_->sibling_next = cc->sibling_next;
305
+ cc->sibling_next->child_->sibling_prev = cc->sibling_prev;
306
+ }
307
+ cc->parent->InternalUnref("child");
308
+ }
309
+
310
+ void Call::CancelWithStatus(grpc_status_code status, const char* description) {
311
+ // copying 'description' is needed to ensure the grpc_call_cancel_with_status
312
+ // guarantee that can be short-lived.
313
+ CancelWithError(grpc_error_set_int(
314
+ grpc_error_set_str(GRPC_ERROR_CREATE(description),
315
+ StatusStrProperty::kGrpcMessage, description),
316
+ StatusIntProperty::kRpcStatus, status));
317
+ }
318
+
319
+ void Call::PropagateCancellationToChildren() {
320
+ ParentCall* pc = parent_call();
321
+ if (pc != nullptr) {
322
+ Call* child;
323
+ MutexLock lock(&pc->child_list_mu);
324
+ child = pc->first_child;
325
+ if (child != nullptr) {
326
+ do {
327
+ Call* next_child_call = child->child_->sibling_next;
328
+ if (child->cancellation_is_inherited_) {
329
+ child->InternalRef("propagate_cancel");
330
+ child->CancelWithError(absl::CancelledError());
331
+ child->InternalUnref("propagate_cancel");
332
+ }
333
+ child = next_child_call;
334
+ } while (child != pc->first_child);
335
+ }
336
+ }
337
+ }
338
+
339
+ char* Call::GetPeer() {
340
+ char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
341
+ if (peer_string != nullptr) return gpr_strdup(peer_string);
342
+ peer_string = grpc_channel_get_target(channel_->c_ptr());
343
+ if (peer_string != nullptr) return peer_string;
344
+ return gpr_strdup("unknown");
345
+ }
346
+
347
+ void Call::DeleteThis() {
348
+ RefCountedPtr<Channel> channel = std::move(channel_);
349
+ Arena* arena = arena_;
350
+ this->~Call();
351
+ channel->UpdateCallSizeEstimate(arena->TotalUsedBytes());
352
+ arena->Destroy();
353
+ }
354
+
355
+ ///////////////////////////////////////////////////////////////////////////////
356
+ // FilterStackCall
357
+ // To be removed once promise conversion is complete
358
+
150
359
  class FilterStackCall final : public Call {
151
360
  public:
152
- ~FilterStackCall() {
361
+ ~FilterStackCall() override {
153
362
  for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
154
363
  if (context_[i].destroy) {
155
364
  context_[i].destroy(context_[i].value);
@@ -184,7 +393,6 @@ class FilterStackCall final : public Call {
184
393
 
185
394
  void CancelWithError(grpc_error_handle error) override;
186
395
  void SetCompletionQueue(grpc_completion_queue* cq) override;
187
- char* GetPeer() override;
188
396
  grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
189
397
  bool is_notify_tag_closure) override;
190
398
  void ExternalRef() override { ext_ref_.Ref(); }
@@ -217,6 +425,13 @@ class FilterStackCall final : public Call {
217
425
  return call_failed_before_recv_message_;
218
426
  }
219
427
 
428
+ absl::string_view GetServerAuthority() const override {
429
+ const Slice* authority_metadata =
430
+ recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
431
+ if (authority_metadata == nullptr) return "";
432
+ return authority_metadata->as_string_view();
433
+ }
434
+
220
435
  grpc_compression_algorithm test_only_compression_algorithm() override {
221
436
  return incoming_compression_algorithm_;
222
437
  }
@@ -235,57 +450,78 @@ class FilterStackCall final : public Call {
235
450
  }
236
451
 
237
452
  private:
238
- // The maximum number of concurrent batches possible.
239
- // Based upon the maximum number of individually queueable ops in the batch
240
- // api:
241
- // - initial metadata send
242
- // - message send
243
- // - status/close send (depending on client/server)
244
- // - initial metadata recv
245
- // - message recv
246
- // - status/close recv (depending on client/server)
247
- static constexpr size_t kMaxConcurrentBatches = 6;
248
-
249
453
  static constexpr gpr_atm kRecvNone = 0;
250
454
  static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
251
455
 
456
+ enum class PendingOp {
457
+ kRecvMessage,
458
+ kRecvInitialMetadata,
459
+ kRecvTrailingMetadata,
460
+ kSends
461
+ };
462
+ static intptr_t PendingOpMask(PendingOp op) {
463
+ return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
464
+ }
465
+ static std::string PendingOpString(intptr_t pending_ops) {
466
+ std::vector<absl::string_view> pending_op_strings;
467
+ if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
468
+ pending_op_strings.push_back("kRecvMessage");
469
+ }
470
+ if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
471
+ pending_op_strings.push_back("kRecvInitialMetadata");
472
+ }
473
+ if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
474
+ pending_op_strings.push_back("kRecvTrailingMetadata");
475
+ }
476
+ if (pending_ops & PendingOpMask(PendingOp::kSends)) {
477
+ pending_op_strings.push_back("kSends");
478
+ }
479
+ return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
480
+ }
252
481
  struct BatchControl {
253
482
  FilterStackCall* call_ = nullptr;
254
483
  grpc_transport_stream_op_batch op_;
255
- /* Share memory for cq_completion and notify_tag as they are never needed
256
- simultaneously. Each byte used in this data structure count as six bytes
257
- per call, so any savings we can make are worthwhile,
484
+ // Share memory for cq_completion and notify_tag as they are never needed
485
+ // simultaneously. Each byte used in this data structure count as six bytes
486
+ // per call, so any savings we can make are worthwhile,
258
487
 
259
- We use notify_tag to determine whether or not to send notification to the
260
- completion queue. Once we've made that determination, we can reuse the
261
- memory for cq_completion. */
488
+ // We use notify_tag to determine whether or not to send notification to the
489
+ // completion queue. Once we've made that determination, we can reuse the
490
+ // memory for cq_completion.
262
491
  union {
263
492
  grpc_cq_completion cq_completion;
264
493
  struct {
265
- /* Any given op indicates completion by either (a) calling a closure or
266
- (b) sending a notification on the call's completion queue. If
267
- \a is_closure is true, \a tag indicates a closure to be invoked;
268
- otherwise, \a tag indicates the tag to be used in the notification to
269
- be sent to the completion queue. */
494
+ // Any given op indicates completion by either (a) calling a closure or
495
+ // (b) sending a notification on the call's completion queue. If
496
+ // \a is_closure is true, \a tag indicates a closure to be invoked;
497
+ // otherwise, \a tag indicates the tag to be used in the notification to
498
+ // be sent to the completion queue.
270
499
  void* tag;
271
500
  bool is_closure;
272
501
  } notify_tag;
273
502
  } completion_data_;
274
503
  grpc_closure start_batch_;
275
504
  grpc_closure finish_batch_;
276
- std::atomic<intptr_t> steps_to_complete_{0};
505
+ std::atomic<intptr_t> ops_pending_{0};
277
506
  AtomicError batch_error_;
278
- void set_num_steps_to_complete(uintptr_t steps) {
279
- steps_to_complete_.store(steps, std::memory_order_release);
507
+ void set_pending_ops(uintptr_t ops) {
508
+ ops_pending_.store(ops, std::memory_order_release);
280
509
  }
281
- bool completed_batch_step() {
282
- return steps_to_complete_.fetch_sub(1, std::memory_order_acq_rel) == 1;
510
+ bool completed_batch_step(PendingOp op) {
511
+ auto mask = PendingOpMask(op);
512
+ auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
513
+ if (grpc_call_trace.enabled()) {
514
+ gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
515
+ PendingOpString(mask).c_str(),
516
+ PendingOpString(r & ~mask).c_str(),
517
+ completion_data_.notify_tag.tag);
518
+ }
519
+ GPR_ASSERT((r & mask) != 0);
520
+ return r == mask;
283
521
  }
284
522
 
285
523
  void PostCompletion();
286
- void FinishStep();
287
- void ContinueReceivingSlices();
288
- void ReceivingSliceReady(grpc_error_handle error);
524
+ void FinishStep(PendingOp op);
289
525
  void ProcessDataAfterMetadata();
290
526
  void ReceivingStreamReady(grpc_error_handle error);
291
527
  void ValidateFilteredMetadata();
@@ -295,9 +531,9 @@ class FilterStackCall final : public Call {
295
531
  };
296
532
 
297
533
  FilterStackCall(Arena* arena, const grpc_call_create_args& args)
298
- : Call(arena, args.server_transport_data == nullptr, args.send_deadline),
534
+ : Call(arena, args.server_transport_data == nullptr, args.send_deadline,
535
+ args.channel->Ref()),
299
536
  cq_(args.cq),
300
- channel_(args.channel),
301
537
  stream_op_payload_(context_) {}
302
538
 
303
539
  static void ReleaseCall(void* call, grpc_error_handle);
@@ -328,60 +564,55 @@ class FilterStackCall final : public Call {
328
564
  CallCombiner call_combiner_;
329
565
  grpc_completion_queue* cq_;
330
566
  grpc_polling_entity pollent_;
331
- grpc_channel* channel_;
332
567
  gpr_cycle_counter start_time_ = gpr_get_cycle_counter();
333
568
 
334
- /** has grpc_call_unref been called */
569
+ /// has grpc_call_unref been called
335
570
  bool destroy_called_ = false;
336
571
  // Trailers-only response status
337
572
  bool is_trailers_only_ = false;
338
- /** which ops are in-flight */
573
+ /// which ops are in-flight
339
574
  bool sent_initial_metadata_ = false;
340
575
  bool sending_message_ = false;
341
576
  bool sent_final_op_ = false;
342
577
  bool received_initial_metadata_ = false;
343
578
  bool receiving_message_ = false;
344
579
  bool requested_final_op_ = false;
345
- gpr_atm any_ops_sent_atm_ = 0;
346
580
  gpr_atm received_final_op_atm_ = 0;
347
581
 
348
582
  BatchControl* active_batches_[kMaxConcurrentBatches] = {};
349
583
  grpc_transport_stream_op_batch_payload stream_op_payload_;
350
584
 
351
- /* first idx: is_receiving, second idx: is_trailing */
585
+ // first idx: is_receiving, second idx: is_trailing
352
586
  grpc_metadata_batch send_initial_metadata_{arena()};
353
587
  grpc_metadata_batch send_trailing_metadata_{arena()};
354
588
  grpc_metadata_batch recv_initial_metadata_{arena()};
355
589
  grpc_metadata_batch recv_trailing_metadata_{arena()};
356
590
 
357
- /* Buffered read metadata waiting to be returned to the application.
358
- Element 0 is initial metadata, element 1 is trailing metadata. */
591
+ // Buffered read metadata waiting to be returned to the application.
592
+ // Element 0 is initial metadata, element 1 is trailing metadata.
359
593
  grpc_metadata_array* buffered_metadata_[2] = {};
360
594
 
361
- // A char* indicating the peer name.
362
- gpr_atm peer_string_ = 0;
363
-
364
- /* Call data useful used for reporting. Only valid after the call has
365
- * completed */
595
+ // Call data useful used for reporting. Only valid after the call has
596
+ // completed
366
597
  grpc_call_final_info final_info_;
367
598
 
368
- /* Compression algorithm for *incoming* data */
599
+ // Compression algorithm for *incoming* data
369
600
  grpc_compression_algorithm incoming_compression_algorithm_ =
370
601
  GRPC_COMPRESS_NONE;
371
- /* Supported encodings (compression algorithms), a bitset.
372
- * Always support no compression. */
602
+ // Supported encodings (compression algorithms), a bitset.
603
+ // Always support no compression.
373
604
  CompressionAlgorithmSet encodings_accepted_by_peer_{GRPC_COMPRESS_NONE};
374
605
 
375
- /* Contexts for various subsystems (security, tracing, ...). */
606
+ // Contexts for various subsystems (security, tracing, ...).
376
607
  grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
377
608
 
378
- ManualConstructor<SliceBufferByteStream> sending_stream_;
609
+ SliceBuffer send_slice_buffer_;
610
+ absl::optional<SliceBuffer> receiving_slice_buffer_;
611
+ uint32_t receiving_stream_flags_;
379
612
 
380
- OrphanablePtr<ByteStream> receiving_stream_;
381
613
  bool call_failed_before_recv_message_ = false;
382
614
  grpc_byte_buffer** receiving_buffer_ = nullptr;
383
615
  grpc_slice receiving_slice_ = grpc_empty_slice();
384
- grpc_closure receiving_slice_ready_;
385
616
  grpc_closure receiving_stream_ready_;
386
617
  grpc_closure receiving_initial_metadata_ready_;
387
618
  grpc_closure receiving_trailing_metadata_ready_;
@@ -406,123 +637,50 @@ class FilterStackCall final : public Call {
406
637
  } final_op_;
407
638
  AtomicError status_error_;
408
639
 
409
- /* recv_state can contain one of the following values:
410
- RECV_NONE : : no initial metadata and messages received
411
- RECV_INITIAL_METADATA_FIRST : received initial metadata first
412
- a batch_control* : received messages first
413
-
414
- +------1------RECV_NONE------3-----+
415
- | |
416
- | |
417
- v v
418
- RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
419
- | ^ | ^
420
- | | | |
421
- +-----2-----+ +-----4-----+
422
-
423
- For 1, 4: See receiving_initial_metadata_ready() function
424
- For 2, 3: See receiving_stream_ready() function */
640
+ // recv_state can contain one of the following values:
641
+ // RECV_NONE : : no initial metadata and messages received
642
+ // RECV_INITIAL_METADATA_FIRST : received initial metadata first
643
+ // a batch_control* : received messages first
644
+
645
+ // +------1------RECV_NONE------3-----+
646
+ // | |
647
+ // | |
648
+ // v v
649
+ // RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
650
+ // | ^ | ^
651
+ // | | | |
652
+ // +-----2-----+ +-----4-----+
653
+
654
+ // For 1, 4: See receiving_initial_metadata_ready() function
655
+ // For 2, 3: See receiving_stream_ready() function
425
656
  gpr_atm recv_state_ = 0;
426
657
  };
427
658
 
428
- Call::ParentCall* Call::GetOrCreateParentCall() {
429
- ParentCall* p = parent_call_.load(std::memory_order_acquire);
430
- if (p == nullptr) {
431
- p = arena_->New<ParentCall>();
432
- ParentCall* expected = nullptr;
433
- if (!parent_call_.compare_exchange_strong(expected, p,
434
- std::memory_order_release,
435
- std::memory_order_relaxed)) {
436
- p->~ParentCall();
437
- p = expected;
438
- }
439
- }
440
- return p;
441
- }
442
-
443
- Call::ParentCall* Call::parent_call() {
444
- return parent_call_.load(std::memory_order_acquire);
445
- }
446
-
447
- absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
448
- child_ = arena()->New<ChildCall>(parent);
449
-
450
- parent->InternalRef("child");
451
- GPR_ASSERT(is_client_);
452
- GPR_ASSERT(!parent->is_client_);
453
-
454
- if (propagation_mask & GRPC_PROPAGATE_DEADLINE) {
455
- send_deadline_ = std::min(send_deadline_, parent->send_deadline_);
456
- }
457
- /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with
458
- * GRPC_PROPAGATE_STATS_CONTEXT */
459
- /* TODO(ctiller): This should change to use the appropriate census start_op
460
- * call. */
461
- if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) {
462
- if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) {
463
- return absl::UnknownError(
464
- "Census tracing propagation requested without Census context "
465
- "propagation");
466
- }
467
- ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
468
- nullptr);
469
- } else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
470
- return absl::UnknownError(
471
- "Census context propagation requested without Census tracing "
472
- "propagation");
473
- }
474
- if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) {
475
- cancellation_is_inherited_ = true;
476
- }
477
- return absl::OkStatus();
478
- }
479
-
480
- void Call::PublishToParent(Call* parent) {
481
- ChildCall* cc = child_;
482
- ParentCall* pc = parent->GetOrCreateParentCall();
483
- MutexLock lock(&pc->child_list_mu);
484
- if (pc->first_child == nullptr) {
485
- pc->first_child = this;
486
- cc->sibling_next = cc->sibling_prev = this;
487
- } else {
488
- cc->sibling_next = pc->first_child;
489
- cc->sibling_prev = pc->first_child->child_->sibling_prev;
490
- cc->sibling_next->child_->sibling_prev =
491
- cc->sibling_prev->child_->sibling_next = this;
492
- }
493
- if (parent->Completed()) {
494
- CancelWithError(GRPC_ERROR_CANCELLED);
495
- }
496
- }
497
-
498
659
  grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
499
660
  grpc_call** out_call) {
500
- GPR_TIMER_SCOPE("grpc_call_create", 0);
501
-
502
- GRPC_CHANNEL_INTERNAL_REF(args->channel, "call");
661
+ Channel* channel = args->channel.get();
503
662
 
504
663
  auto add_init_error = [](grpc_error_handle* composite,
505
664
  grpc_error_handle new_err) {
506
- if (new_err == GRPC_ERROR_NONE) return;
507
- if (*composite == GRPC_ERROR_NONE) {
508
- *composite = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Call creation failed");
665
+ if (new_err.ok()) return;
666
+ if (composite->ok()) {
667
+ *composite = GRPC_ERROR_CREATE("Call creation failed");
509
668
  }
510
669
  *composite = grpc_error_add_child(*composite, new_err);
511
670
  };
512
671
 
513
672
  Arena* arena;
514
673
  FilterStackCall* call;
515
- grpc_error_handle error = GRPC_ERROR_NONE;
516
- grpc_channel_stack* channel_stack =
517
- grpc_channel_get_channel_stack(args->channel);
518
- size_t initial_size = grpc_channel_get_call_size_estimate(args->channel);
519
- GRPC_STATS_INC_CALL_INITIAL_SIZE(initial_size);
674
+ grpc_error_handle error;
675
+ grpc_channel_stack* channel_stack = channel->channel_stack();
676
+ size_t initial_size = channel->CallSizeEstimate();
677
+ global_stats().IncrementCallInitialSize(initial_size);
520
678
  size_t call_alloc_size =
521
679
  GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
522
680
  channel_stack->call_stack_size;
523
681
 
524
682
  std::pair<Arena*, void*> arena_with_call = Arena::CreateWithAlloc(
525
- initial_size, call_alloc_size, &*args->channel->allocator);
683
+ initial_size, call_alloc_size, channel->allocator());
526
684
  arena = arena_with_call.first;
527
685
  call = new (arena_with_call.second) FilterStackCall(arena, *args);
528
686
  GPR_DEBUG_ASSERT(FromC(call->c_ptr()) == call);
@@ -533,8 +691,8 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
533
691
  call->final_op_.client.status_details = nullptr;
534
692
  call->final_op_.client.status = nullptr;
535
693
  call->final_op_.client.error_string = nullptr;
536
- GRPC_STATS_INC_CLIENT_CALLS_CREATED();
537
- path = grpc_slice_ref_internal(args->path->c_slice());
694
+ global_stats().IncrementClientCallsCreated();
695
+ path = CSliceRef(args->path->c_slice());
538
696
  call->send_initial_metadata_.Set(HttpPathMetadata(),
539
697
  std::move(*args->path));
540
698
  if (args->authority.has_value()) {
@@ -542,7 +700,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
542
700
  std::move(*args->authority));
543
701
  }
544
702
  } else {
545
- GRPC_STATS_INC_SERVER_CALLS_CREATED();
703
+ global_stats().IncrementServerCallsCreated();
546
704
  call->final_op_.server.cancelled = nullptr;
547
705
  call->final_op_.server.core_server = args->server;
548
706
  }
@@ -552,7 +710,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
552
710
  add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
553
711
  parent, args->propagation_mask)));
554
712
  }
555
- /* initial refcount dropped by grpc_call_unref */
713
+ // initial refcount dropped by grpc_call_unref
556
714
  grpc_call_element_args call_args = {
557
715
  call->call_stack(), args->server_transport_data,
558
716
  call->context_, path,
@@ -565,8 +723,8 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
565
723
  call->PublishToParent(parent);
566
724
  }
567
725
 
568
- if (error != GRPC_ERROR_NONE) {
569
- call->CancelWithError(GRPC_ERROR_REF(error));
726
+ if (!error.ok()) {
727
+ call->CancelWithError(error);
570
728
  }
571
729
  if (args->cq != nullptr) {
572
730
  GPR_ASSERT(args->pollset_set_alternative == nullptr &&
@@ -586,8 +744,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
586
744
  }
587
745
 
588
746
  if (call->is_client()) {
589
- channelz::ChannelNode* channelz_channel =
590
- grpc_channel_get_channelz_node(call->channel_);
747
+ channelz::ChannelNode* channelz_channel = channel->channelz_node();
591
748
  if (channelz_channel != nullptr) {
592
749
  channelz_channel->RecordCallStarted();
593
750
  }
@@ -599,7 +756,7 @@ grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
599
756
  }
600
757
  }
601
758
 
602
- grpc_slice_unref_internal(path);
759
+ CSliceUnref(path);
603
760
 
604
761
  return error;
605
762
  }
@@ -608,8 +765,7 @@ void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
608
765
  GPR_ASSERT(cq);
609
766
 
610
767
  if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
611
- gpr_log(GPR_ERROR, "A pollset_set is already registered for this call.");
612
- abort();
768
+ Crash("A pollset_set is already registered for this call.");
613
769
  }
614
770
  cq_ = cq;
615
771
  GRPC_CQ_INTERNAL_REF(cq, "bind");
@@ -618,20 +774,14 @@ void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
618
774
  }
619
775
 
620
776
  void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
621
- auto* c = static_cast<FilterStackCall*>(call);
622
- grpc_channel* channel = c->channel_;
623
- Arena* arena = c->arena();
624
- c->~FilterStackCall();
625
- grpc_channel_update_call_size_estimate(channel, arena->Destroy());
626
- GRPC_CHANNEL_INTERNAL_UNREF(channel, "call");
777
+ static_cast<FilterStackCall*>(call)->DeleteThis();
627
778
  }
628
779
 
629
780
  void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
630
- GPR_TIMER_SCOPE("destroy_call", 0);
631
781
  auto* c = static_cast<FilterStackCall*>(call);
632
782
  c->recv_initial_metadata_.Clear();
633
783
  c->recv_trailing_metadata_.Clear();
634
- c->receiving_stream_.reset();
784
+ c->receiving_slice_buffer_.reset();
635
785
  ParentCall* pc = c->parent_call();
636
786
  if (pc != nullptr) {
637
787
  pc->~ParentCall();
@@ -644,7 +794,7 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
644
794
  grpc_error_get_status(status_error, c->send_deadline(),
645
795
  &c->final_info_.final_status, nullptr, nullptr,
646
796
  &(c->final_info_.error_string));
647
- c->status_error_.set(GRPC_ERROR_NONE);
797
+ c->status_error_.set(absl::OkStatus());
648
798
  c->final_info_.stats.latency =
649
799
  gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time_);
650
800
  grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
@@ -652,30 +802,9 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
652
802
  grpc_schedule_on_exec_ctx));
653
803
  }
654
804
 
655
- void Call::MaybeUnpublishFromParent() {
656
- ChildCall* cc = child_;
657
- if (cc == nullptr) return;
658
-
659
- ParentCall* pc = cc->parent->parent_call();
660
- {
661
- MutexLock lock(&pc->child_list_mu);
662
- if (this == pc->first_child) {
663
- pc->first_child = cc->sibling_next;
664
- if (this == pc->first_child) {
665
- pc->first_child = nullptr;
666
- }
667
- }
668
- cc->sibling_prev->child_->sibling_next = cc->sibling_next;
669
- cc->sibling_next->child_->sibling_prev = cc->sibling_prev;
670
- }
671
- cc->parent->InternalUnref("child");
672
- }
673
-
674
805
  void FilterStackCall::ExternalUnref() {
675
806
  if (GPR_LIKELY(!ext_ref_.Unref())) return;
676
807
 
677
- GPR_TIMER_SCOPE("grpc_call_unref", 0);
678
-
679
808
  ApplicationCallbackExecCtx callback_exec_ctx;
680
809
  ExecCtx exec_ctx;
681
810
 
@@ -685,10 +814,9 @@ void FilterStackCall::ExternalUnref() {
685
814
 
686
815
  GPR_ASSERT(!destroy_called_);
687
816
  destroy_called_ = true;
688
- bool cancel = gpr_atm_acq_load(&any_ops_sent_atm_) != 0 &&
689
- gpr_atm_acq_load(&received_final_op_atm_) == 0;
817
+ bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
690
818
  if (cancel) {
691
- CancelWithError(GRPC_ERROR_CANCELLED);
819
+ CancelWithError(absl::CancelledError());
692
820
  } else {
693
821
  // Unset the call combiner cancellation closure. This has the
694
822
  // effect of scheduling the previously set cancellation closure, if
@@ -699,14 +827,6 @@ void FilterStackCall::ExternalUnref() {
699
827
  InternalUnref("destroy");
700
828
  }
701
829
 
702
- char* FilterStackCall::GetPeer() {
703
- char* peer_string = reinterpret_cast<char*>(gpr_atm_acq_load(&peer_string_));
704
- if (peer_string != nullptr) return gpr_strdup(peer_string);
705
- peer_string = grpc_channel_get_target(channel_);
706
- if (peer_string != nullptr) return peer_string;
707
- return gpr_strdup("unknown");
708
- }
709
-
710
830
  // start_batch_closure points to a caller-allocated closure to be used
711
831
  // for entering the call combiner.
712
832
  void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
@@ -714,7 +834,6 @@ void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
714
834
  // This is called via the call combiner to start sending a batch down
715
835
  // the filter stack.
716
836
  auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
717
- GPR_TIMER_SCOPE("execute_batch_in_call_combiner", 0);
718
837
  grpc_transport_stream_op_batch* batch =
719
838
  static_cast<grpc_transport_stream_op_batch*>(arg);
720
839
  auto* call =
@@ -727,7 +846,7 @@ void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
727
846
  GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
728
847
  grpc_schedule_on_exec_ctx);
729
848
  GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
730
- GRPC_ERROR_NONE, "executing batch");
849
+ absl::OkStatus(), "executing batch");
731
850
  }
732
851
 
733
852
  namespace {
@@ -750,15 +869,15 @@ static void done_termination(void* arg, grpc_error_handle /*error*/) {
750
869
 
751
870
  void FilterStackCall::CancelWithError(grpc_error_handle error) {
752
871
  if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
753
- GRPC_ERROR_UNREF(error);
754
872
  return;
755
873
  }
874
+ ClearPeerString();
756
875
  InternalRef("termination");
757
876
  // Inform the call combiner of the cancellation, so that it can cancel
758
877
  // any in-flight asynchronous actions that may be holding the call
759
878
  // combiner. This ensures that the cancel_stream batch can be sent
760
879
  // down the filter stack in a timely manner.
761
- call_combiner_.Cancel(GRPC_ERROR_REF(error));
880
+ call_combiner_.Cancel(error);
762
881
  CancelState* state = new CancelState;
763
882
  state->call = this;
764
883
  GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
@@ -770,19 +889,10 @@ void FilterStackCall::CancelWithError(grpc_error_handle error) {
770
889
  ExecuteBatch(op, &state->start_batch);
771
890
  }
772
891
 
773
- void Call::CancelWithStatus(grpc_status_code status, const char* description) {
774
- // copying 'description' is needed to ensure the grpc_call_cancel_with_status
775
- // guarantee that can be short-lived.
776
- CancelWithError(grpc_error_set_int(
777
- grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description),
778
- GRPC_ERROR_STR_GRPC_MESSAGE, description),
779
- GRPC_ERROR_INT_GRPC_STATUS, status));
780
- }
781
-
782
892
  void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
783
893
  if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
784
894
  gpr_log(GPR_DEBUG, "set_final_status %s", is_client() ? "CLI" : "SVR");
785
- gpr_log(GPR_DEBUG, "%s", grpc_error_std_string(error).c_str());
895
+ gpr_log(GPR_DEBUG, "%s", StatusToString(error).c_str());
786
896
  }
787
897
  if (is_client()) {
788
898
  std::string status_details;
@@ -792,9 +902,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
792
902
  *final_op_.client.status_details =
793
903
  grpc_slice_from_cpp_string(std::move(status_details));
794
904
  status_error_.set(error);
795
- GRPC_ERROR_UNREF(error);
796
- channelz::ChannelNode* channelz_channel =
797
- grpc_channel_get_channelz_node(channel_);
905
+ channelz::ChannelNode* channelz_channel = channel()->channelz_node();
798
906
  if (channelz_channel != nullptr) {
799
907
  if (*final_op_.client.status != GRPC_STATUS_OK) {
800
908
  channelz_channel->RecordCallFailed();
@@ -804,7 +912,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
804
912
  }
805
913
  } else {
806
914
  *final_op_.server.cancelled =
807
- error != GRPC_ERROR_NONE || !sent_server_trailing_metadata_;
915
+ !error.ok() || !sent_server_trailing_metadata_;
808
916
  channelz::ServerNode* channelz_node =
809
917
  final_op_.server.core_server->channelz_node();
810
918
  if (channelz_node != nullptr) {
@@ -814,7 +922,6 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
814
922
  channelz_node->RecordCallSucceeded();
815
923
  }
816
924
  }
817
- GRPC_ERROR_UNREF(error);
818
925
  }
819
926
  }
820
927
 
@@ -840,8 +947,7 @@ bool FilterStackCall::PrepareApplicationMetadata(size_t count,
840
947
  // Filter "content-length metadata"
841
948
  continue;
842
949
  }
843
- batch->Append(StringViewFromSlice(md->key),
844
- Slice(grpc_slice_ref_internal(md->value)),
950
+ batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
845
951
  [md](absl::string_view error, const Slice& value) {
846
952
  gpr_log(GPR_DEBUG, "Append error: %s",
847
953
  absl::StrCat("key=", StringViewFromSlice(md->key),
@@ -914,7 +1020,6 @@ void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
914
1020
  if (b->count() == 0) return;
915
1021
  if (!is_client() && is_trailing) return;
916
1022
  if (is_trailing && buffered_metadata_[1] == nullptr) return;
917
- GPR_TIMER_SCOPE("publish_app_metadata", 0);
918
1023
  grpc_metadata_array* dest;
919
1024
  dest = buffered_metadata_[is_trailing];
920
1025
  if (dest->count + b->count() > dest->capacity) {
@@ -938,39 +1043,37 @@ void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
938
1043
 
939
1044
  void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
940
1045
  grpc_error_handle batch_error) {
941
- if (batch_error != GRPC_ERROR_NONE) {
1046
+ if (!batch_error.ok()) {
942
1047
  SetFinalStatus(batch_error);
943
1048
  } else {
944
1049
  absl::optional<grpc_status_code> grpc_status =
945
1050
  b->Take(GrpcStatusMetadata());
946
1051
  if (grpc_status.has_value()) {
947
1052
  grpc_status_code status_code = *grpc_status;
948
- grpc_error_handle error = GRPC_ERROR_NONE;
1053
+ grpc_error_handle error;
949
1054
  if (status_code != GRPC_STATUS_OK) {
950
1055
  char* peer = GetPeer();
951
1056
  error = grpc_error_set_int(
952
- GRPC_ERROR_CREATE_FROM_CPP_STRING(
953
- absl::StrCat("Error received from peer ", peer)),
954
- GRPC_ERROR_INT_GRPC_STATUS, static_cast<intptr_t>(status_code));
1057
+ GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ", peer)),
1058
+ StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
955
1059
  gpr_free(peer);
956
1060
  }
957
1061
  auto grpc_message = b->Take(GrpcMessageMetadata());
958
1062
  if (grpc_message.has_value()) {
959
- error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
1063
+ error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
960
1064
  grpc_message->as_string_view());
961
- } else if (error != GRPC_ERROR_NONE) {
962
- error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "");
1065
+ } else if (!error.ok()) {
1066
+ error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
963
1067
  }
964
- SetFinalStatus(GRPC_ERROR_REF(error));
965
- GRPC_ERROR_UNREF(error);
1068
+ SetFinalStatus(error);
966
1069
  } else if (!is_client()) {
967
- SetFinalStatus(GRPC_ERROR_NONE);
1070
+ SetFinalStatus(absl::OkStatus());
968
1071
  } else {
969
1072
  gpr_log(GPR_DEBUG,
970
1073
  "Received trailing metadata with no error and no status");
971
- SetFinalStatus(grpc_error_set_int(
972
- GRPC_ERROR_CREATE_FROM_STATIC_STRING("No status received"),
973
- GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNKNOWN));
1074
+ SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
1075
+ StatusIntProperty::kRpcStatus,
1076
+ GRPC_STATUS_UNKNOWN));
974
1077
  }
975
1078
  }
976
1079
  PublishAppMetadata(b, true);
@@ -978,7 +1081,7 @@ void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
978
1081
 
979
1082
  namespace {
980
1083
  bool AreWriteFlagsValid(uint32_t flags) {
981
- /* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */
1084
+ // check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
982
1085
  const uint32_t allowed_write_positions =
983
1086
  (GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
984
1087
  const uint32_t invalid_positions = ~allowed_write_positions;
@@ -986,7 +1089,7 @@ bool AreWriteFlagsValid(uint32_t flags) {
986
1089
  }
987
1090
 
988
1091
  bool AreInitialMetadataFlagsValid(uint32_t flags) {
989
- /* check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set */
1092
+ // check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
990
1093
  uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
991
1094
  return !(flags & invalid_positions);
992
1095
  }
@@ -1034,68 +1137,48 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
1034
1137
  return bctl;
1035
1138
  }
1036
1139
 
1037
- void Call::PropagateCancellationToChildren() {
1038
- ParentCall* pc = parent_call();
1039
- if (pc != nullptr) {
1040
- Call* child;
1041
- MutexLock lock(&pc->child_list_mu);
1042
- child = pc->first_child;
1043
- if (child != nullptr) {
1044
- do {
1045
- Call* next_child_call = child->child_->sibling_next;
1046
- if (child->cancellation_is_inherited_) {
1047
- child->InternalRef("propagate_cancel");
1048
- child->CancelWithError(GRPC_ERROR_CANCELLED);
1049
- child->InternalUnref("propagate_cancel");
1050
- }
1051
- child = next_child_call;
1052
- } while (child != pc->first_child);
1053
- }
1054
- }
1055
- }
1056
-
1057
1140
  void FilterStackCall::BatchControl::PostCompletion() {
1058
1141
  FilterStackCall* call = call_;
1059
- grpc_error_handle error = GRPC_ERROR_REF(batch_error_.get());
1142
+ grpc_error_handle error = batch_error_.get();
1143
+ if (grpc_call_trace.enabled()) {
1144
+ gpr_log(GPR_DEBUG, "tag:%p batch_error=%s", completion_data_.notify_tag.tag,
1145
+ error.ToString().c_str());
1146
+ }
1060
1147
 
1061
1148
  if (op_.send_initial_metadata) {
1062
1149
  call->send_initial_metadata_.Clear();
1063
1150
  }
1064
1151
  if (op_.send_message) {
1065
- if (op_.payload->send_message.stream_write_closed &&
1066
- error == GRPC_ERROR_NONE) {
1152
+ if (op_.payload->send_message.stream_write_closed && error.ok()) {
1067
1153
  error = grpc_error_add_child(
1068
- error, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
1154
+ error, GRPC_ERROR_CREATE(
1069
1155
  "Attempt to send message after stream was closed."));
1070
1156
  }
1071
1157
  call->sending_message_ = false;
1158
+ call->send_slice_buffer_.Clear();
1072
1159
  }
1073
1160
  if (op_.send_trailing_metadata) {
1074
1161
  call->send_trailing_metadata_.Clear();
1075
1162
  }
1076
1163
  if (op_.recv_trailing_metadata) {
1077
- /* propagate cancellation to any interested children */
1164
+ // propagate cancellation to any interested children
1078
1165
  gpr_atm_rel_store(&call->received_final_op_atm_, 1);
1079
1166
  call->PropagateCancellationToChildren();
1080
- GRPC_ERROR_UNREF(error);
1081
- error = GRPC_ERROR_NONE;
1167
+ error = absl::OkStatus();
1082
1168
  }
1083
- if (error != GRPC_ERROR_NONE && op_.recv_message &&
1084
- *call->receiving_buffer_ != nullptr) {
1169
+ if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
1085
1170
  grpc_byte_buffer_destroy(*call->receiving_buffer_);
1086
1171
  *call->receiving_buffer_ = nullptr;
1087
1172
  }
1088
- batch_error_.set(GRPC_ERROR_NONE);
1173
+ batch_error_.set(absl::OkStatus());
1089
1174
 
1090
1175
  if (completion_data_.notify_tag.is_closure) {
1091
- /* unrefs error */
1092
1176
  call_ = nullptr;
1093
1177
  Closure::Run(DEBUG_LOCATION,
1094
1178
  static_cast<grpc_closure*>(completion_data_.notify_tag.tag),
1095
1179
  error);
1096
1180
  call->InternalUnref("completion");
1097
1181
  } else {
1098
- /* unrefs error */
1099
1182
  grpc_cq_end_op(
1100
1183
  call->cq_, completion_data_.notify_tag.tag, error,
1101
1184
  [](void* user_data, grpc_cq_completion* /*storage*/) {
@@ -1108,118 +1191,58 @@ void FilterStackCall::BatchControl::PostCompletion() {
1108
1191
  }
1109
1192
  }
1110
1193
 
1111
- void FilterStackCall::BatchControl::FinishStep() {
1112
- if (GPR_UNLIKELY(completed_batch_step())) {
1194
+ void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
1195
+ if (GPR_UNLIKELY(completed_batch_step(op))) {
1113
1196
  PostCompletion();
1114
1197
  }
1115
1198
  }
1116
1199
 
1117
- void FilterStackCall::BatchControl::ContinueReceivingSlices() {
1118
- grpc_error_handle error;
1119
- FilterStackCall* call = call_;
1120
- for (;;) {
1121
- size_t remaining = call->receiving_stream_->length() -
1122
- (*call->receiving_buffer_)->data.raw.slice_buffer.length;
1123
- if (remaining == 0) {
1124
- call->receiving_message_ = false;
1125
- call->receiving_stream_.reset();
1126
- FinishStep();
1127
- return;
1128
- }
1129
- if (call->receiving_stream_->Next(remaining,
1130
- &call->receiving_slice_ready_)) {
1131
- error = call->receiving_stream_->Pull(&call->receiving_slice_);
1132
- if (error == GRPC_ERROR_NONE) {
1133
- grpc_slice_buffer_add(
1134
- &(*call->receiving_buffer_)->data.raw.slice_buffer,
1135
- call->receiving_slice_);
1136
- } else {
1137
- call->receiving_stream_.reset();
1138
- grpc_byte_buffer_destroy(*call->receiving_buffer_);
1139
- *call->receiving_buffer_ = nullptr;
1140
- call->receiving_message_ = false;
1141
- FinishStep();
1142
- GRPC_ERROR_UNREF(error);
1143
- return;
1144
- }
1145
- } else {
1146
- return;
1147
- }
1148
- }
1149
- }
1150
-
1151
- void FilterStackCall::BatchControl::ReceivingSliceReady(
1152
- grpc_error_handle error) {
1153
- FilterStackCall* call = call_;
1154
- bool release_error = false;
1155
-
1156
- if (error == GRPC_ERROR_NONE) {
1157
- grpc_slice slice;
1158
- error = call->receiving_stream_->Pull(&slice);
1159
- if (error == GRPC_ERROR_NONE) {
1160
- grpc_slice_buffer_add(&(*call->receiving_buffer_)->data.raw.slice_buffer,
1161
- slice);
1162
- ContinueReceivingSlices();
1163
- } else {
1164
- /* Error returned by ByteStream::Pull() needs to be released manually */
1165
- release_error = true;
1166
- }
1167
- }
1168
-
1169
- if (error != GRPC_ERROR_NONE) {
1170
- if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
1171
- GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error));
1172
- }
1173
- call->receiving_stream_.reset();
1174
- grpc_byte_buffer_destroy(*call->receiving_buffer_);
1175
- *call->receiving_buffer_ = nullptr;
1176
- call->receiving_message_ = false;
1177
- FinishStep();
1178
- if (release_error) {
1179
- GRPC_ERROR_UNREF(error);
1180
- }
1181
- }
1182
- }
1183
-
1184
1200
  void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
1185
1201
  FilterStackCall* call = call_;
1186
- if (call->receiving_stream_ == nullptr) {
1202
+ if (!call->receiving_slice_buffer_.has_value()) {
1187
1203
  *call->receiving_buffer_ = nullptr;
1188
1204
  call->receiving_message_ = false;
1189
- FinishStep();
1205
+ FinishStep(PendingOp::kRecvMessage);
1190
1206
  } else {
1191
- call->test_only_last_message_flags_ = call->receiving_stream_->flags();
1192
- if ((call->receiving_stream_->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
1207
+ call->test_only_last_message_flags_ = call->receiving_stream_flags_;
1208
+ if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
1193
1209
  (call->incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) {
1194
1210
  *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
1195
1211
  nullptr, 0, call->incoming_compression_algorithm_);
1196
1212
  } else {
1197
1213
  *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
1198
1214
  }
1199
- GRPC_CLOSURE_INIT(
1200
- &call->receiving_slice_ready_,
1201
- [](void* bctl, grpc_error_handle error) {
1202
- static_cast<BatchControl*>(bctl)->ReceivingSliceReady(error);
1203
- },
1204
- this, grpc_schedule_on_exec_ctx);
1205
- ContinueReceivingSlices();
1215
+ grpc_slice_buffer_move_into(
1216
+ call->receiving_slice_buffer_->c_slice_buffer(),
1217
+ &(*call->receiving_buffer_)->data.raw.slice_buffer);
1218
+ call->receiving_message_ = false;
1219
+ call->receiving_slice_buffer_.reset();
1220
+ FinishStep(PendingOp::kRecvMessage);
1206
1221
  }
1207
1222
  }
1208
1223
 
1209
1224
  void FilterStackCall::BatchControl::ReceivingStreamReady(
1210
1225
  grpc_error_handle error) {
1226
+ if (grpc_call_trace.enabled()) {
1227
+ gpr_log(GPR_DEBUG,
1228
+ "tag:%p ReceivingStreamReady error=%s "
1229
+ "receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
1230
+ completion_data_.notify_tag.tag, error.ToString().c_str(),
1231
+ call_->receiving_slice_buffer_.has_value(),
1232
+ gpr_atm_no_barrier_load(&call_->recv_state_));
1233
+ }
1211
1234
  FilterStackCall* call = call_;
1212
- if (error != GRPC_ERROR_NONE) {
1213
- call->receiving_stream_.reset();
1235
+ if (!error.ok()) {
1236
+ call->receiving_slice_buffer_.reset();
1214
1237
  if (batch_error_.ok()) {
1215
1238
  batch_error_.set(error);
1216
1239
  }
1217
- call->CancelWithError(GRPC_ERROR_REF(error));
1240
+ call->CancelWithError(error);
1218
1241
  }
1219
- /* If recv_state is kRecvNone, we will save the batch_control
1220
- * object with rel_cas, and will not use it after the cas. Its corresponding
1221
- * acq_load is in receiving_initial_metadata_ready() */
1222
- if (error != GRPC_ERROR_NONE || call->receiving_stream_ == nullptr ||
1242
+ // If recv_state is kRecvNone, we will save the batch_control
1243
+ // object with rel_cas, and will not use it after the cas. Its corresponding
1244
+ // acq_load is in receiving_initial_metadata_ready()
1245
+ if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
1223
1246
  !gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
1224
1247
  reinterpret_cast<gpr_atm>(this))) {
1225
1248
  ProcessDataAfterMetadata();
@@ -1243,23 +1266,24 @@ void FilterStackCall::HandleCompressionAlgorithmNotAccepted(
1243
1266
  gpr_log(GPR_ERROR,
1244
1267
  "Compression algorithm ('%s') not present in the "
1245
1268
  "accepted encodings (%s)",
1246
- algo_name, encodings_accepted_by_peer_.ToString().c_str());
1269
+ algo_name,
1270
+ std::string(encodings_accepted_by_peer_.ToString()).c_str());
1247
1271
  }
1248
1272
 
1249
1273
  void FilterStackCall::BatchControl::ValidateFilteredMetadata() {
1250
1274
  FilterStackCall* call = call_;
1251
1275
 
1252
1276
  const grpc_compression_options compression_options =
1253
- grpc_channel_compression_options(call->channel_);
1277
+ call->channel()->compression_options();
1254
1278
  const grpc_compression_algorithm compression_algorithm =
1255
1279
  call->incoming_compression_algorithm_;
1256
1280
  if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
1257
1281
  compression_options.enabled_algorithms_bitset)
1258
1282
  .IsSet(compression_algorithm))) {
1259
- /* check if algorithm is supported by current channel config */
1283
+ // check if algorithm is supported by current channel config
1260
1284
  call->HandleCompressionAlgorithmDisabled(compression_algorithm);
1261
1285
  }
1262
- /* GRPC_COMPRESS_NONE is always set. */
1286
+ // GRPC_COMPRESS_NONE is always set.
1263
1287
  GPR_DEBUG_ASSERT(call->encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
1264
1288
  if (GPR_UNLIKELY(
1265
1289
  !call->encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
@@ -1275,12 +1299,11 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
1275
1299
 
1276
1300
  GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
1277
1301
 
1278
- if (error == GRPC_ERROR_NONE) {
1302
+ if (error.ok()) {
1279
1303
  grpc_metadata_batch* md = &call->recv_initial_metadata_;
1280
1304
  call->RecvInitialFilter(md);
1281
1305
 
1282
- /* TODO(ctiller): this could be moved into recv_initial_filter now */
1283
- GPR_TIMER_SCOPE("validate_filtered_metadata", 0);
1306
+ // TODO(ctiller): this could be moved into recv_initial_filter now
1284
1307
  ValidateFilteredMetadata();
1285
1308
 
1286
1309
  absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
@@ -1291,41 +1314,41 @@ void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
1291
1314
  if (batch_error_.ok()) {
1292
1315
  batch_error_.set(error);
1293
1316
  }
1294
- call->CancelWithError(GRPC_ERROR_REF(error));
1317
+ call->CancelWithError(error);
1295
1318
  }
1296
1319
 
1297
1320
  grpc_closure* saved_rsr_closure = nullptr;
1298
1321
  while (true) {
1299
1322
  gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
1300
- /* Should only receive initial metadata once */
1323
+ // Should only receive initial metadata once
1301
1324
  GPR_ASSERT(rsr_bctlp != 1);
1302
1325
  if (rsr_bctlp == 0) {
1303
- /* We haven't seen initial metadata and messages before, thus initial
1304
- * metadata is received first.
1305
- * no_barrier_cas is used, as this function won't access the batch_control
1306
- * object saved by receiving_stream_ready() if the initial metadata is
1307
- * received first. */
1326
+ // We haven't seen initial metadata and messages before, thus initial
1327
+ // metadata is received first.
1328
+ // no_barrier_cas is used, as this function won't access the batch_control
1329
+ // object saved by receiving_stream_ready() if the initial metadata is
1330
+ // received first.
1308
1331
  if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
1309
1332
  kRecvInitialMetadataFirst)) {
1310
1333
  break;
1311
1334
  }
1312
1335
  } else {
1313
- /* Already received messages */
1336
+ // Already received messages
1314
1337
  saved_rsr_closure = GRPC_CLOSURE_CREATE(
1315
1338
  [](void* bctl, grpc_error_handle error) {
1316
1339
  static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
1317
1340
  },
1318
1341
  reinterpret_cast<BatchControl*>(rsr_bctlp),
1319
1342
  grpc_schedule_on_exec_ctx);
1320
- /* No need to modify recv_state */
1343
+ // No need to modify recv_state
1321
1344
  break;
1322
1345
  }
1323
1346
  }
1324
1347
  if (saved_rsr_closure != nullptr) {
1325
- Closure::Run(DEBUG_LOCATION, saved_rsr_closure, GRPC_ERROR_REF(error));
1348
+ Closure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
1326
1349
  }
1327
1350
 
1328
- FinishStep();
1351
+ FinishStep(PendingOp::kRecvInitialMetadata);
1329
1352
  }
1330
1353
 
1331
1354
  void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
@@ -1333,8 +1356,8 @@ void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
1333
1356
  GRPC_CALL_COMBINER_STOP(call_->call_combiner(),
1334
1357
  "recv_trailing_metadata_ready");
1335
1358
  grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
1336
- call_->RecvTrailingFilter(md, GRPC_ERROR_REF(error));
1337
- FinishStep();
1359
+ call_->RecvTrailingFilter(md, error);
1360
+ FinishStep(PendingOp::kRecvTrailingMetadata);
1338
1361
  }
1339
1362
 
1340
1363
  void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
@@ -1342,26 +1365,41 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
1342
1365
  if (batch_error_.ok()) {
1343
1366
  batch_error_.set(error);
1344
1367
  }
1345
- if (error != GRPC_ERROR_NONE) {
1346
- call_->CancelWithError(GRPC_ERROR_REF(error));
1368
+ if (!error.ok()) {
1369
+ call_->CancelWithError(error);
1370
+ }
1371
+ FinishStep(PendingOp::kSends);
1372
+ }
1373
+
1374
+ namespace {
1375
+ void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag,
1376
+ bool is_notify_tag_closure) {
1377
+ if (!is_notify_tag_closure) {
1378
+ GPR_ASSERT(grpc_cq_begin_op(cq, notify_tag));
1379
+ grpc_cq_end_op(
1380
+ cq, notify_tag, absl::OkStatus(),
1381
+ [](void*, grpc_cq_completion* completion) { gpr_free(completion); },
1382
+ nullptr,
1383
+ static_cast<grpc_cq_completion*>(
1384
+ gpr_malloc(sizeof(grpc_cq_completion))));
1385
+ } else {
1386
+ Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
1387
+ absl::OkStatus());
1347
1388
  }
1348
- FinishStep();
1349
1389
  }
1390
+ } // namespace
1350
1391
 
1351
1392
  grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1352
1393
  void* notify_tag,
1353
1394
  bool is_notify_tag_closure) {
1354
- GPR_TIMER_SCOPE("call_start_batch", 0);
1355
-
1356
1395
  size_t i;
1357
1396
  const grpc_op* op;
1358
1397
  BatchControl* bctl;
1359
- bool has_send_ops = false;
1360
- int num_recv_ops = 0;
1361
1398
  grpc_call_error error = GRPC_CALL_OK;
1362
1399
  grpc_transport_stream_op_batch* stream_op;
1363
1400
  grpc_transport_stream_op_batch_payload* stream_op_payload;
1364
1401
  uint32_t seen_ops = 0;
1402
+ intptr_t pending_ops = 0;
1365
1403
 
1366
1404
  for (i = 0; i < nops; i++) {
1367
1405
  if (seen_ops & (1u << ops[i].op)) {
@@ -1370,21 +1408,19 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1370
1408
  seen_ops |= (1u << ops[i].op);
1371
1409
  }
1372
1410
 
1411
+ if (!is_client() &&
1412
+ (seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
1413
+ (seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
1414
+ gpr_log(GPR_ERROR,
1415
+ "******************* SEND_STATUS WITH RECV_MESSAGE "
1416
+ "*******************");
1417
+ return GRPC_CALL_ERROR;
1418
+ }
1419
+
1373
1420
  GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
1374
1421
 
1375
1422
  if (nops == 0) {
1376
- if (!is_notify_tag_closure) {
1377
- GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag));
1378
- grpc_cq_end_op(
1379
- cq_, notify_tag, GRPC_ERROR_NONE,
1380
- [](void*, grpc_cq_completion* completion) { gpr_free(completion); },
1381
- nullptr,
1382
- static_cast<grpc_cq_completion*>(
1383
- gpr_malloc(sizeof(grpc_cq_completion))));
1384
- } else {
1385
- Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
1386
- GRPC_ERROR_NONE);
1387
- }
1423
+ EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
1388
1424
  error = GRPC_CALL_OK;
1389
1425
  goto done;
1390
1426
  }
@@ -1400,7 +1436,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1400
1436
  stream_op = &bctl->op_;
1401
1437
  stream_op_payload = &stream_op_payload_;
1402
1438
 
1403
- /* rewrite batch ops into a transport op */
1439
+ // rewrite batch ops into a transport op
1404
1440
  for (i = 0; i < nops; i++) {
1405
1441
  op = &ops[i];
1406
1442
  if (op->reserved != nullptr) {
@@ -1409,7 +1445,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1409
1445
  }
1410
1446
  switch (op->op) {
1411
1447
  case GRPC_OP_SEND_INITIAL_METADATA: {
1412
- /* Flag validation: currently allow no flags */
1448
+ // Flag validation: currently allow no flags
1413
1449
  if (!AreInitialMetadataFlagsValid(op->flags)) {
1414
1450
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1415
1451
  goto done_with_error;
@@ -1422,7 +1458,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1422
1458
  // algorithm by setting the initial metadata with key of
1423
1459
  // GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY, we shouldn't override that
1424
1460
  // with the compression algorithm mapped from compression level.
1425
- /* process compression level */
1461
+ // process compression level
1426
1462
  grpc_compression_level effective_compression_level =
1427
1463
  GRPC_COMPRESS_LEVEL_NONE;
1428
1464
  bool level_set = false;
@@ -1432,7 +1468,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1432
1468
  level_set = true;
1433
1469
  } else {
1434
1470
  const grpc_compression_options copts =
1435
- grpc_channel_compression_options(channel_);
1471
+ channel()->compression_options();
1436
1472
  if (copts.default_level.is_set) {
1437
1473
  level_set = true;
1438
1474
  effective_compression_level = copts.default_level.level;
@@ -1462,18 +1498,25 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1462
1498
  }
1463
1499
  // Ignore any te metadata key value pairs specified.
1464
1500
  send_initial_metadata_.Remove(TeMetadata());
1465
- /* TODO(ctiller): just make these the same variable? */
1501
+ // TODO(ctiller): just make these the same variable?
1466
1502
  if (is_client() && send_deadline() != Timestamp::InfFuture()) {
1467
1503
  send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
1468
1504
  }
1505
+ if (is_client()) {
1506
+ send_initial_metadata_.Set(
1507
+ WaitForReady(),
1508
+ WaitForReady::ValueType{
1509
+ (op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
1510
+ (op->flags &
1511
+ GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
1512
+ }
1469
1513
  stream_op_payload->send_initial_metadata.send_initial_metadata =
1470
1514
  &send_initial_metadata_;
1471
- stream_op_payload->send_initial_metadata.send_initial_metadata_flags =
1472
- op->flags;
1473
1515
  if (is_client()) {
1474
- stream_op_payload->send_initial_metadata.peer_string = &peer_string_;
1516
+ stream_op_payload->send_initial_metadata.peer_string =
1517
+ peer_string_atm_ptr();
1475
1518
  }
1476
- has_send_ops = true;
1519
+ pending_ops |= PendingOpMask(PendingOp::kSends);
1477
1520
  break;
1478
1521
  }
1479
1522
  case GRPC_OP_SEND_MESSAGE: {
@@ -1490,24 +1533,26 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1490
1533
  goto done_with_error;
1491
1534
  }
1492
1535
  uint32_t flags = op->flags;
1493
- /* If the outgoing buffer is already compressed, mark it as so in the
1494
- flags. These will be picked up by the compression filter and further
1495
- (wasteful) attempts at compression skipped. */
1536
+ // If the outgoing buffer is already compressed, mark it as so in the
1537
+ // flags. These will be picked up by the compression filter and further
1538
+ // (wasteful) attempts at compression skipped.
1496
1539
  if (op->data.send_message.send_message->data.raw.compression >
1497
1540
  GRPC_COMPRESS_NONE) {
1498
1541
  flags |= GRPC_WRITE_INTERNAL_COMPRESS;
1499
1542
  }
1500
1543
  stream_op->send_message = true;
1501
1544
  sending_message_ = true;
1502
- sending_stream_.Init(
1503
- &op->data.send_message.send_message->data.raw.slice_buffer, flags);
1504
- stream_op_payload->send_message.send_message.reset(
1505
- sending_stream_.get());
1506
- has_send_ops = true;
1545
+ send_slice_buffer_.Clear();
1546
+ grpc_slice_buffer_move_into(
1547
+ &op->data.send_message.send_message->data.raw.slice_buffer,
1548
+ send_slice_buffer_.c_slice_buffer());
1549
+ stream_op_payload->send_message.flags = flags;
1550
+ stream_op_payload->send_message.send_message = &send_slice_buffer_;
1551
+ pending_ops |= PendingOpMask(PendingOp::kSends);
1507
1552
  break;
1508
1553
  }
1509
1554
  case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
1510
- /* Flag validation: currently allow no flags */
1555
+ // Flag validation: currently allow no flags
1511
1556
  if (op->flags != 0) {
1512
1557
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1513
1558
  goto done_with_error;
@@ -1524,11 +1569,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1524
1569
  sent_final_op_ = true;
1525
1570
  stream_op_payload->send_trailing_metadata.send_trailing_metadata =
1526
1571
  &send_trailing_metadata_;
1527
- has_send_ops = true;
1572
+ pending_ops |= PendingOpMask(PendingOp::kSends);
1528
1573
  break;
1529
1574
  }
1530
1575
  case GRPC_OP_SEND_STATUS_FROM_SERVER: {
1531
- /* Flag validation: currently allow no flags */
1576
+ // Flag validation: currently allow no flags
1532
1577
  if (op->flags != 0) {
1533
1578
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1534
1579
  goto done_with_error;
@@ -1558,11 +1603,10 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1558
1603
 
1559
1604
  grpc_error_handle status_error =
1560
1605
  op->data.send_status_from_server.status == GRPC_STATUS_OK
1561
- ? GRPC_ERROR_NONE
1606
+ ? absl::OkStatus()
1562
1607
  : grpc_error_set_int(
1563
- GRPC_ERROR_CREATE_FROM_STATIC_STRING(
1564
- "Server returned error"),
1565
- GRPC_ERROR_INT_GRPC_STATUS,
1608
+ GRPC_ERROR_CREATE("Server returned error"),
1609
+ StatusIntProperty::kRpcStatus,
1566
1610
  static_cast<intptr_t>(
1567
1611
  op->data.send_status_from_server.status));
1568
1612
  if (op->data.send_status_from_server.status_details != nullptr) {
@@ -1570,16 +1614,15 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1570
1614
  GrpcMessageMetadata(),
1571
1615
  Slice(grpc_slice_copy(
1572
1616
  *op->data.send_status_from_server.status_details)));
1573
- if (status_error != GRPC_ERROR_NONE) {
1617
+ if (!status_error.ok()) {
1574
1618
  status_error = grpc_error_set_str(
1575
- status_error, GRPC_ERROR_STR_GRPC_MESSAGE,
1619
+ status_error, StatusStrProperty::kGrpcMessage,
1576
1620
  StringViewFromSlice(
1577
1621
  *op->data.send_status_from_server.status_details));
1578
1622
  }
1579
1623
  }
1580
1624
 
1581
1625
  status_error_.set(status_error);
1582
- GRPC_ERROR_UNREF(status_error);
1583
1626
 
1584
1627
  send_trailing_metadata_.Set(GrpcStatusMetadata(),
1585
1628
  op->data.send_status_from_server.status);
@@ -1590,11 +1633,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1590
1633
  &send_trailing_metadata_;
1591
1634
  stream_op_payload->send_trailing_metadata.sent =
1592
1635
  &sent_server_trailing_metadata_;
1593
- has_send_ops = true;
1636
+ pending_ops |= PendingOpMask(PendingOp::kSends);
1594
1637
  break;
1595
1638
  }
1596
1639
  case GRPC_OP_RECV_INITIAL_METADATA: {
1597
- /* Flag validation: currently allow no flags */
1640
+ // Flag validation: currently allow no flags
1598
1641
  if (op->flags != 0) {
1599
1642
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1600
1643
  goto done_with_error;
@@ -1622,13 +1665,14 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1622
1665
  stream_op_payload->recv_initial_metadata.trailing_metadata_available =
1623
1666
  &is_trailers_only_;
1624
1667
  } else {
1625
- stream_op_payload->recv_initial_metadata.peer_string = &peer_string_;
1668
+ stream_op_payload->recv_initial_metadata.peer_string =
1669
+ peer_string_atm_ptr();
1626
1670
  }
1627
- ++num_recv_ops;
1671
+ pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
1628
1672
  break;
1629
1673
  }
1630
1674
  case GRPC_OP_RECV_MESSAGE: {
1631
- /* Flag validation: currently allow no flags */
1675
+ // Flag validation: currently allow no flags
1632
1676
  if (op->flags != 0) {
1633
1677
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1634
1678
  goto done_with_error;
@@ -1639,8 +1683,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1639
1683
  }
1640
1684
  receiving_message_ = true;
1641
1685
  stream_op->recv_message = true;
1686
+ receiving_slice_buffer_.reset();
1642
1687
  receiving_buffer_ = op->data.recv_message.recv_message;
1643
- stream_op_payload->recv_message.recv_message = &receiving_stream_;
1688
+ stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
1689
+ receiving_stream_flags_ = 0;
1690
+ stream_op_payload->recv_message.flags = &receiving_stream_flags_;
1644
1691
  stream_op_payload->recv_message.call_failed_before_recv_message =
1645
1692
  &call_failed_before_recv_message_;
1646
1693
  GRPC_CLOSURE_INIT(
@@ -1657,11 +1704,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1657
1704
  bctl, grpc_schedule_on_exec_ctx);
1658
1705
  stream_op_payload->recv_message.recv_message_ready =
1659
1706
  &receiving_stream_ready_;
1660
- ++num_recv_ops;
1707
+ pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
1661
1708
  break;
1662
1709
  }
1663
1710
  case GRPC_OP_RECV_STATUS_ON_CLIENT: {
1664
- /* Flag validation: currently allow no flags */
1711
+ // Flag validation: currently allow no flags
1665
1712
  if (op->flags != 0) {
1666
1713
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1667
1714
  goto done_with_error;
@@ -1696,11 +1743,11 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1696
1743
  bctl, grpc_schedule_on_exec_ctx);
1697
1744
  stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
1698
1745
  &receiving_trailing_metadata_ready_;
1699
- ++num_recv_ops;
1746
+ pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
1700
1747
  break;
1701
1748
  }
1702
1749
  case GRPC_OP_RECV_CLOSE_ON_SERVER: {
1703
- /* Flag validation: currently allow no flags */
1750
+ // Flag validation: currently allow no flags
1704
1751
  if (op->flags != 0) {
1705
1752
  error = GRPC_CALL_ERROR_INVALID_FLAGS;
1706
1753
  goto done_with_error;
@@ -1729,7 +1776,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1729
1776
  bctl, grpc_schedule_on_exec_ctx);
1730
1777
  stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
1731
1778
  &receiving_trailing_metadata_ready_;
1732
- ++num_recv_ops;
1779
+ pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
1733
1780
  break;
1734
1781
  }
1735
1782
  }
@@ -1739,9 +1786,9 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1739
1786
  if (!is_notify_tag_closure) {
1740
1787
  GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag));
1741
1788
  }
1742
- bctl->set_num_steps_to_complete((has_send_ops ? 1 : 0) + num_recv_ops);
1789
+ bctl->set_pending_ops(pending_ops);
1743
1790
 
1744
- if (has_send_ops) {
1791
+ if (pending_ops & PendingOpMask(PendingOp::kSends)) {
1745
1792
  GRPC_CLOSURE_INIT(
1746
1793
  &bctl->finish_batch_,
1747
1794
  [](void* bctl, grpc_error_handle error) {
@@ -1751,24 +1798,26 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1751
1798
  stream_op->on_complete = &bctl->finish_batch_;
1752
1799
  }
1753
1800
 
1754
- gpr_atm_rel_store(&any_ops_sent_atm_, 1);
1801
+ if (grpc_call_trace.enabled()) {
1802
+ gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
1803
+ PendingOpString(pending_ops).c_str(),
1804
+ grpc_transport_stream_op_batch_string(stream_op).c_str(),
1805
+ bctl->completion_data_.notify_tag.tag);
1806
+ }
1807
+
1755
1808
  ExecuteBatch(stream_op, &bctl->start_batch_);
1756
1809
 
1757
1810
  done:
1758
1811
  return error;
1759
1812
 
1760
1813
  done_with_error:
1761
- /* reverse any mutations that occurred */
1814
+ // reverse any mutations that occurred
1762
1815
  if (stream_op->send_initial_metadata) {
1763
1816
  sent_initial_metadata_ = false;
1764
1817
  send_initial_metadata_.Clear();
1765
1818
  }
1766
1819
  if (stream_op->send_message) {
1767
1820
  sending_message_ = false;
1768
- // No need to invoke call->sending_stream->Orphan() explicitly.
1769
- // stream_op_payload->send_message.send_message.reset() calls Deletor
1770
- // of call->sending_stream which in-turn invokes the Orphan() method.
1771
- stream_op_payload->send_message.send_message.reset();
1772
1821
  }
1773
1822
  if (stream_op->send_trailing_metadata) {
1774
1823
  sent_final_op_ = false;
@@ -1795,8 +1844,1689 @@ void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
1795
1844
  context_[elem].destroy = destroy;
1796
1845
  }
1797
1846
 
1847
+ ///////////////////////////////////////////////////////////////////////////////
1848
+ // Metadata validation helpers
1849
+
1850
+ namespace {
1851
+ bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
1852
+ for (size_t i = 0; i < count; i++) {
1853
+ grpc_metadata* md = &metadata[i];
1854
+ if (!GRPC_LOG_IF_ERROR("validate_metadata",
1855
+ grpc_validate_header_key_is_legal(md->key))) {
1856
+ return false;
1857
+ } else if (!grpc_is_binary_header_internal(md->key) &&
1858
+ !GRPC_LOG_IF_ERROR(
1859
+ "validate_metadata",
1860
+ grpc_validate_header_nonbin_value_is_legal(md->value))) {
1861
+ return false;
1862
+ } else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
1863
+ // HTTP2 hpack encoding has a maximum limit.
1864
+ return false;
1865
+ }
1866
+ }
1867
+ return true;
1868
+ }
1869
+ } // namespace
1870
+
1871
+ ///////////////////////////////////////////////////////////////////////////////
1872
+ // PromiseBasedCall
1873
+ // Will be folded into Call once the promise conversion is done
1874
+
1875
+ class PromiseBasedCall : public Call,
1876
+ public Activity,
1877
+ public Wakeable,
1878
+ public grpc_event_engine::experimental::EventEngine::
1879
+ Closure /* for deadlines */ {
1880
+ public:
1881
+ PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
1882
+ const grpc_call_create_args& args);
1883
+
1884
+ void ContextSet(grpc_context_index elem, void* value,
1885
+ void (*destroy)(void* value)) override;
1886
+ void* ContextGet(grpc_context_index elem) const override;
1887
+ void SetCompletionQueue(grpc_completion_queue* cq) override;
1888
+ void SetCompletionQueueLocked(grpc_completion_queue* cq)
1889
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
1890
+ void CancelWithError(absl::Status error) final ABSL_LOCKS_EXCLUDED(mu_) {
1891
+ MutexLock lock(&mu_);
1892
+ CancelWithErrorLocked(std::move(error));
1893
+ }
1894
+ virtual void CancelWithErrorLocked(absl::Status error)
1895
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
1896
+ bool Completed() final ABSL_LOCKS_EXCLUDED(mu_) {
1897
+ MutexLock lock(&mu_);
1898
+ return completed_;
1899
+ }
1900
+
1901
+ void Orphan() final {
1902
+ MutexLock lock(&mu_);
1903
+ if (!completed_) {
1904
+ CancelWithErrorLocked(absl::CancelledError("Call orphaned"));
1905
+ }
1906
+ }
1907
+
1908
+ // Implementation of call refcounting: move this to DualRefCounted once we
1909
+ // don't need to maintain FilterStackCall compatibility
1910
+ void ExternalRef() final {
1911
+ const uint64_t prev_ref_pair =
1912
+ refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed);
1913
+ if (grpc_call_refcount_trace.enabled()) {
1914
+ gpr_log(GPR_DEBUG, "%s EXTERNAL_REF: %d:%d->%d:%d", DebugTag().c_str(),
1915
+ GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1916
+ GetStrongRefs(prev_ref_pair) + 1, GetWeakRefs(prev_ref_pair));
1917
+ }
1918
+ }
1919
+ void ExternalUnref() final {
1920
+ const uint64_t prev_ref_pair =
1921
+ refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel);
1922
+ if (grpc_call_refcount_trace.enabled()) {
1923
+ gpr_log(GPR_DEBUG, "%s EXTERNAL_UNREF: %d:%d->%d:%d", DebugTag().c_str(),
1924
+ GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1925
+ GetStrongRefs(prev_ref_pair) - 1, GetWeakRefs(prev_ref_pair) + 1);
1926
+ }
1927
+ const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
1928
+ if (GPR_UNLIKELY(strong_refs == 1)) {
1929
+ Orphan();
1930
+ }
1931
+ // Now drop the weak ref.
1932
+ InternalUnref("external_ref");
1933
+ }
1934
+ void InternalRef(const char* reason) final {
1935
+ uint64_t n = refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed);
1936
+ if (grpc_call_refcount_trace.enabled()) {
1937
+ gpr_log(GPR_DEBUG, "%s REF: %s %d:%d->%d:%d", DebugTag().c_str(), reason,
1938
+ GetStrongRefs(n), GetWeakRefs(n), GetStrongRefs(n),
1939
+ GetWeakRefs(n) + 1);
1940
+ }
1941
+ }
1942
+ void InternalUnref(const char* reason) final {
1943
+ const uint64_t prev_ref_pair =
1944
+ refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel);
1945
+ if (grpc_call_refcount_trace.enabled()) {
1946
+ gpr_log(GPR_DEBUG, "%s UNREF: %s %d:%d->%d:%d", DebugTag().c_str(),
1947
+ reason, GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair),
1948
+ GetStrongRefs(prev_ref_pair), GetWeakRefs(prev_ref_pair) - 1);
1949
+ }
1950
+ if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) {
1951
+ DeleteThis();
1952
+ }
1953
+ }
1954
+
1955
+ // Activity methods
1956
+ void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
1957
+ Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override {
1958
+ InternalRef("wakeup");
1959
+ // If ASAN is defined, we leverage it to detect dropped Waker objects.
1960
+ // Usually Waker must be destroyed or woken up, but (especially with arenas)
1961
+ // it's not uncommon to create a Waker and then do neither. In that case it's
1962
+ // incredibly fraught to diagnose where the dropped reference to this object was
1963
+ // created. Instead, leverage ASAN and create a new object per expected wakeup.
1964
+ // Now when we drop such an object ASAN will fail and we'll get a callstack to
1965
+ // the creation of the waker in question.
1966
+ #if defined(__has_feature)
1967
+ #if __has_feature(address_sanitizer)
1968
+ #define GRPC_CALL_USES_ASAN_WAKER
1969
+ class AsanWaker final : public Wakeable {
1970
+ public:
1971
+ explicit AsanWaker(PromiseBasedCall* call) : call_(call) {}
1972
+
1973
+ void Wakeup(void*) override {
1974
+ call_->Wakeup(nullptr);
1975
+ delete this;
1976
+ }
1977
+
1978
+ void Drop(void*) override {
1979
+ call_->Drop(nullptr);
1980
+ delete this;
1981
+ }
1982
+
1983
+ std::string ActivityDebugTag(void*) const override {
1984
+ return call_->DebugTag();
1985
+ }
1986
+
1987
+ private:
1988
+ PromiseBasedCall* call_;
1989
+ };
1990
+ return Waker(new AsanWaker(this), nullptr);
1991
+ #endif
1992
+ #endif
1993
+ #ifndef GRPC_CALL_USES_ASAN_WAKER
1994
+ return Waker(this, nullptr);
1995
+ #endif
1996
+ }
1997
+ Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override;
1998
+
1999
+ // Wakeable methods
2000
+ void Wakeup(void*) override {
2001
+ channel()->event_engine()->Run([this] {
2002
+ ApplicationCallbackExecCtx app_exec_ctx;
2003
+ ExecCtx exec_ctx;
2004
+ {
2005
+ ScopedContext activity_context(this);
2006
+ MutexLock lock(&mu_);
2007
+ Update();
2008
+ }
2009
+ InternalUnref("wakeup");
2010
+ });
2011
+ }
2012
+ void Drop(void*) override { InternalUnref("wakeup"); }
2013
+
2014
+ void RunInContext(absl::AnyInvocable<void()> fn) {
2015
+ if (Activity::current() == this) {
2016
+ fn();
2017
+ } else {
2018
+ InternalRef("in_context");
2019
+ channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable {
2020
+ ApplicationCallbackExecCtx app_exec_ctx;
2021
+ ExecCtx exec_ctx;
2022
+ {
2023
+ ScopedContext activity_context(this);
2024
+ MutexLock lock(&mu_);
2025
+ fn();
2026
+ Update();
2027
+ }
2028
+ InternalUnref("in_context");
2029
+ });
2030
+ }
2031
+ }
2032
+
2033
+ grpc_compression_algorithm test_only_compression_algorithm() override {
2034
+ abort();
2035
+ }
2036
+ uint32_t test_only_message_flags() override { abort(); }
2037
+ uint32_t test_only_encodings_accepted_by_peer() override { abort(); }
2038
+ grpc_compression_algorithm compression_for_level(
2039
+ grpc_compression_level) override {
2040
+ abort();
2041
+ }
2042
+
2043
+ // This should return nullptr for the promise stack (and alternative means
2044
+ // for that functionality be invented)
2045
+ grpc_call_stack* call_stack() override { return nullptr; }
2046
+
2047
+ void UpdateDeadline(Timestamp deadline);
2048
+ void ResetDeadline();
2049
+
2050
+ // Implementation of EventEngine::Closure, called when deadline expires
2051
+ void Run() override;
2052
+
2053
+ virtual ServerCallContext* server_call_context() { return nullptr; }
2054
+
2055
+ protected:
2056
+ class ScopedContext
2057
+ : public ScopedActivity,
2058
+ public promise_detail::Context<Arena>,
2059
+ public promise_detail::Context<grpc_call_context_element>,
2060
+ public promise_detail::Context<CallContext>,
2061
+ public promise_detail::Context<CallFinalization> {
2062
+ public:
2063
+ explicit ScopedContext(PromiseBasedCall* call)
2064
+ : ScopedActivity(call),
2065
+ promise_detail::Context<Arena>(call->arena()),
2066
+ promise_detail::Context<grpc_call_context_element>(call->context_),
2067
+ promise_detail::Context<CallContext>(&call->call_context_),
2068
+ promise_detail::Context<CallFinalization>(&call->finalization_) {}
2069
+ };
2070
+
2071
+ class Completion {
2072
+ public:
2073
+ Completion() : index_(kNullIndex) {}
2074
+ ~Completion() { GPR_ASSERT(index_ == kNullIndex); }
2075
+ explicit Completion(uint8_t index) : index_(index) {}
2076
+ Completion(const Completion& other) = delete;
2077
+ Completion& operator=(const Completion& other) = delete;
2078
+ Completion(Completion&& other) noexcept : index_(other.index_) {
2079
+ other.index_ = kNullIndex;
2080
+ }
2081
+ Completion& operator=(Completion&& other) noexcept {
2082
+ GPR_ASSERT(index_ == kNullIndex);
2083
+ index_ = other.index_;
2084
+ other.index_ = kNullIndex;
2085
+ return *this;
2086
+ }
2087
+
2088
+ uint8_t index() const { return index_; }
2089
+ uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
2090
+ bool has_value() const { return index_ != kNullIndex; }
2091
+
2092
+ private:
2093
+ enum : uint8_t { kNullIndex = 0xff };
2094
+ uint8_t index_;
2095
+ };
2096
+
2097
+ ~PromiseBasedCall() override {
2098
+ if (non_owning_wakeable_) non_owning_wakeable_->DropActivity();
2099
+ if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
2100
+ }
2101
+
2102
+ // Enumerates why a Completion is still pending
2103
+ enum class PendingOp {
2104
+ // We're in the midst of starting a batch of operations
2105
+ kStartingBatch = 0,
2106
+ // The following correspond with the batch operations from above
2107
+ kReceiveInitialMetadata,
2108
+ kReceiveStatusOnClient,
2109
+ kReceiveCloseOnServer = kReceiveStatusOnClient,
2110
+ kSendMessage,
2111
+ kReceiveMessage,
2112
+ kSendStatusFromServer,
2113
+ kSendCloseFromClient = kSendStatusFromServer,
2114
+ };
2115
+
2116
+ const char* PendingOpString(PendingOp reason) const {
2117
+ switch (reason) {
2118
+ case PendingOp::kStartingBatch:
2119
+ return "StartingBatch";
2120
+ case PendingOp::kReceiveInitialMetadata:
2121
+ return "ReceiveInitialMetadata";
2122
+ case PendingOp::kReceiveStatusOnClient:
2123
+ return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
2124
+ case PendingOp::kSendMessage:
2125
+ return "SendMessage";
2126
+ case PendingOp::kReceiveMessage:
2127
+ return "ReceiveMessage";
2128
+ case PendingOp::kSendStatusFromServer:
2129
+ return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
2130
+ }
2131
+ return "Unknown";
2132
+ }
2133
+
2134
+ static constexpr uint8_t PendingOpBit(PendingOp reason) {
2135
+ return 1 << static_cast<int>(reason);
2136
+ }
2137
+
2138
+ Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; }
2139
+ // Begin work on a completion, recording the tag/closure to notify.
2140
+ // Use the op selected in \a ops to determine the index to allocate into.
2141
+ // Starts the "StartingBatch" PendingOp immediately.
2142
+ // Assumes at least one operation in \a ops.
2143
+ Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops)
2144
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2145
+ // Add one pending op to the completion, and return it.
2146
+ Completion AddOpToCompletion(const Completion& completion, PendingOp reason)
2147
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2148
+ // Stringify a completion
2149
+ std::string CompletionString(const Completion& completion) const {
2150
+ return completion.has_value()
2151
+ ? absl::StrFormat(
2152
+ "%d:tag=%p", static_cast<int>(completion.index()),
2153
+ completion_info_[completion.index()].pending.tag)
2154
+ : "no-completion";
2155
+ }
2156
+ // Finish one op on the completion. Must have been previously been added.
2157
+ // The completion as a whole finishes when all pending ops finish.
2158
+ void FinishOpOnCompletion(Completion* completion, PendingOp reason)
2159
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2160
+ // Mark the completion as failed. Does not finish it.
2161
+ void FailCompletion(const Completion& completion,
2162
+ SourceLocation source_location = {});
2163
+ // Run the promise polling loop until it stalls.
2164
+ void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2165
+ // Update the promise state once.
2166
+ virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
2167
+ // Accept the stats from the context (call once we have proof the transport is
2168
+ // done with them).
2169
+ // Right now this means that promise based calls do not record correct stats
2170
+ // with census if they are cancelled.
2171
+ // TODO(ctiller): this should be remedied before promise based calls are
2172
+ // dexperimentalized.
2173
+ void AcceptTransportStatsFromContext() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2174
+ final_stats_ = *call_context_.call_stats();
2175
+ }
2176
+
2177
+ grpc_completion_queue* cq() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return cq_; }
2178
+
2179
+ void CToMetadata(grpc_metadata* metadata, size_t count,
2180
+ grpc_metadata_batch* batch);
2181
+
2182
+ std::string ActivityDebugTag(void*) const override { return DebugTag(); }
2183
+
2184
+ // At the end of the call run any finalization actions.
2185
+ void RunFinalization(grpc_status_code status, const char* status_details) {
2186
+ grpc_call_final_info final_info;
2187
+ final_info.stats = final_stats_;
2188
+ final_info.final_status = status;
2189
+ final_info.error_string = status_details;
2190
+ finalization_.Run(&final_info);
2191
+ }
2192
+
2193
+ std::string PresentAndCompletionText(const char* caption, bool has,
2194
+ const Completion& completion) const {
2195
+ if (has) {
2196
+ if (completion.has_value()) {
2197
+ return absl::StrCat(caption, ":", CompletionString(completion), " ");
2198
+ } else {
2199
+ return absl::StrCat(caption,
2200
+ ":!!BUG:operation is present, no completion!! ");
2201
+ }
2202
+ } else {
2203
+ if (!completion.has_value()) {
2204
+ return "";
2205
+ } else {
2206
+ return absl::StrCat(caption, ":no-op:", CompletionString(completion),
2207
+ " ");
2208
+ }
2209
+ }
2210
+ }
2211
+
2212
+ std::string PollStateDebugString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2213
+ return absl::StrCat(PresentAndCompletionText("outstanding_send",
2214
+ outstanding_send_.has_value(),
2215
+ send_message_completion_)
2216
+ .c_str(),
2217
+ PresentAndCompletionText("outstanding_recv",
2218
+ outstanding_recv_.has_value(),
2219
+ recv_message_completion_)
2220
+ .c_str());
2221
+ }
2222
+
2223
+ void StartRecvMessage(const grpc_op& op, const Completion& completion,
2224
+ PipeReceiver<MessageHandle>* receiver)
2225
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2226
+ void PollRecvMessage(grpc_compression_algorithm compression_algorithm)
2227
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2228
+ void CancelRecvMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2229
+ void StartSendMessage(const grpc_op& op, const Completion& completion,
2230
+ PipeSender<MessageHandle>* sender)
2231
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2232
+ bool PollSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2233
+ void CancelSendMessage() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
2234
+
2235
+ bool completed() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2236
+ return completed_;
2237
+ }
2238
+ void set_completed() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { completed_ = true; }
2239
+ bool is_sending() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
2240
+ return outstanding_send_.has_value();
2241
+ }
2242
+
2243
+ private:
2244
+ union CompletionInfo {
2245
+ struct Pending {
2246
+ // Bitmask of PendingOps
2247
+ uint8_t pending_op_bits;
2248
+ bool is_closure;
2249
+ bool success;
2250
+ void* tag;
2251
+ } pending;
2252
+ grpc_cq_completion completion;
2253
+ };
2254
+
2255
+ class NonOwningWakable final : public Wakeable {
2256
+ public:
2257
+ explicit NonOwningWakable(PromiseBasedCall* call) : call_(call) {}
2258
+
2259
+ // Ref the Handle (not the activity).
2260
+ void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
2261
+
2262
+ // Activity is going away... drop its reference and sever the connection
2263
+ // back.
2264
+ void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) {
2265
+ auto unref = absl::MakeCleanup([this]() { Unref(); });
2266
+ MutexLock lock(&mu_);
2267
+ GPR_ASSERT(call_ != nullptr);
2268
+ call_ = nullptr;
2269
+ }
2270
+
2271
+ // Activity needs to wake up (if it still exists!) - wake it up, and drop
2272
+ // the ref that was kept for this handle.
2273
+ void Wakeup(void*) override ABSL_LOCKS_EXCLUDED(mu_) {
2274
+ // Drop the ref to the handle at end of scope (we have one ref = one
2275
+ // wakeup semantics).
2276
+ auto unref = absl::MakeCleanup([this]() { Unref(); });
2277
+ ReleasableMutexLock lock(&mu_);
2278
+ // Note that activity refcount can drop to zero, but we could win the lock
2279
+ // against DropActivity, so we need to only increase activities refcount
2280
+ // if it is non-zero.
2281
+ PromiseBasedCall* call = call_;
2282
+ if (call != nullptr && call->RefIfNonZero()) {
2283
+ lock.Release();
2284
+ // Activity still exists and we have a reference: wake it up, which will
2285
+ // drop the ref.
2286
+ call->Wakeup(nullptr);
2287
+ }
2288
+ }
2289
+
2290
+ std::string ActivityDebugTag(void*) const override {
2291
+ MutexLock lock(&mu_);
2292
+ return call_ == nullptr ? "<unknown>" : call_->DebugTag();
2293
+ }
2294
+
2295
+ void Drop(void*) override { Unref(); }
2296
+
2297
+ private:
2298
+ // Unref the Handle (not the activity).
2299
+ void Unref() {
2300
+ if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) {
2301
+ delete this;
2302
+ }
2303
+ }
2304
+
2305
+ mutable Mutex mu_;
2306
+ // We have two initial refs: one for the wakeup that this is created for,
2307
+ // and will be dropped by Wakeup, and the other for the activity which is
2308
+ // dropped by DropActivity.
2309
+ std::atomic<size_t> refs_{2};
2310
+ PromiseBasedCall* call_ ABSL_GUARDED_BY(mu_);
2311
+ };
2312
+
2313
+ static void OnDestroy(void* arg, grpc_error_handle) {
2314
+ auto* call = static_cast<PromiseBasedCall*>(arg);
2315
+ ScopedContext context(call);
2316
+ call->DeleteThis();
2317
+ }
2318
+
2319
+ // First 32 bits are strong refs, next 32 bits are weak refs.
2320
+ static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) {
2321
+ return (static_cast<uint64_t>(strong) << 32) + static_cast<int64_t>(weak);
2322
+ }
2323
+ static uint32_t GetStrongRefs(uint64_t ref_pair) {
2324
+ return static_cast<uint32_t>(ref_pair >> 32);
2325
+ }
2326
+ static uint32_t GetWeakRefs(uint64_t ref_pair) {
2327
+ return static_cast<uint32_t>(ref_pair & 0xffffffffu);
2328
+ }
2329
+
2330
+ bool RefIfNonZero() {
2331
+ uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire);
2332
+ do {
2333
+ const uint32_t strong_refs = GetStrongRefs(prev_ref_pair);
2334
+ if (strong_refs == 0) return false;
2335
+ } while (!refs_.compare_exchange_weak(
2336
+ prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0),
2337
+ std::memory_order_acq_rel, std::memory_order_acquire));
2338
+ return true;
2339
+ }
2340
+
2341
+ mutable Mutex mu_;
2342
+ std::atomic<uint64_t> refs_;
2343
+ CallContext call_context_{this};
2344
+ bool keep_polling_ ABSL_GUARDED_BY(mu()) = false;
2345
+
2346
+ // Contexts for various subsystems (security, tracing, ...).
2347
+ grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
2348
+ grpc_completion_queue* cq_ ABSL_GUARDED_BY(mu_);
2349
+ NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr;
2350
+ CompletionInfo completion_info_[6];
2351
+ grpc_call_stats final_stats_{};
2352
+ CallFinalization finalization_;
2353
+ // Current deadline.
2354
+ Timestamp deadline_ = Timestamp::InfFuture();
2355
+ grpc_event_engine::experimental::EventEngine::TaskHandle deadline_task_;
2356
+ absl::optional<PipeSender<MessageHandle>::PushType> outstanding_send_
2357
+ ABSL_GUARDED_BY(mu_);
2358
+ absl::optional<PipeReceiverNextType<MessageHandle>> outstanding_recv_
2359
+ ABSL_GUARDED_BY(mu_);
2360
+ grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu_) = nullptr;
2361
+ Completion send_message_completion_ ABSL_GUARDED_BY(mu_);
2362
+ Completion recv_message_completion_ ABSL_GUARDED_BY(mu_);
2363
+ bool completed_ ABSL_GUARDED_BY(mu_) = false;
2364
+ };
2365
+
2366
+ template <typename T>
2367
+ grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
2368
+ grpc_call** out_call) {
2369
+ Channel* channel = args->channel.get();
2370
+
2371
+ auto alloc = Arena::CreateWithAlloc(channel->CallSizeEstimate(), sizeof(T),
2372
+ channel->allocator());
2373
+ PromiseBasedCall* call = new (alloc.second) T(alloc.first, args);
2374
+ *out_call = call->c_ptr();
2375
+ GPR_DEBUG_ASSERT(Call::FromC(*out_call) == call);
2376
+ return absl::OkStatus();
2377
+ }
2378
+
2379
+ PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
2380
+ const grpc_call_create_args& args)
2381
+ : Call(arena, args.server_transport_data == nullptr, args.send_deadline,
2382
+ args.channel->Ref()),
2383
+ refs_(MakeRefPair(initial_external_refs, 0)),
2384
+ cq_(args.cq) {
2385
+ if (args.cq != nullptr) {
2386
+ GPR_ASSERT(args.pollset_set_alternative == nullptr &&
2387
+ "Only one of 'cq' and 'pollset_set_alternative' should be "
2388
+ "non-nullptr.");
2389
+ GRPC_CQ_INTERNAL_REF(args.cq, "bind");
2390
+ call_context_.pollent_ =
2391
+ grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args.cq));
2392
+ }
2393
+ if (args.pollset_set_alternative != nullptr) {
2394
+ call_context_.pollent_ = grpc_polling_entity_create_from_pollset_set(
2395
+ args.pollset_set_alternative);
2396
+ }
2397
+ }
2398
+
2399
+ Waker PromiseBasedCall::MakeNonOwningWaker() {
2400
+ if (non_owning_wakeable_ == nullptr) {
2401
+ non_owning_wakeable_ = new NonOwningWakable(this);
2402
+ } else {
2403
+ non_owning_wakeable_->Ref();
2404
+ }
2405
+ return Waker(non_owning_wakeable_, nullptr);
2406
+ }
2407
+
2408
+ void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count,
2409
+ grpc_metadata_batch* b) {
2410
+ for (size_t i = 0; i < count; i++) {
2411
+ grpc_metadata* md = &metadata[i];
2412
+ auto key = StringViewFromSlice(md->key);
2413
+ // Filter "content-length metadata"
2414
+ if (key == "content-length") continue;
2415
+ b->Append(key, Slice(CSliceRef(md->value)),
2416
+ [md](absl::string_view error, const Slice& value) {
2417
+ gpr_log(GPR_DEBUG, "Append error: %s",
2418
+ absl::StrCat("key=", StringViewFromSlice(md->key),
2419
+ " error=", error,
2420
+ " value=", value.as_string_view())
2421
+ .c_str());
2422
+ });
2423
+ }
2424
+ }
2425
+
2426
+ void PromiseBasedCall::ContextSet(grpc_context_index elem, void* value,
2427
+ void (*destroy)(void*)) {
2428
+ if (context_[elem].destroy != nullptr) {
2429
+ context_[elem].destroy(context_[elem].value);
2430
+ }
2431
+ context_[elem].value = value;
2432
+ context_[elem].destroy = destroy;
2433
+ }
2434
+
2435
+ void* PromiseBasedCall::ContextGet(grpc_context_index elem) const {
2436
+ return context_[elem].value;
2437
+ }
2438
+
2439
+ PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
2440
+ void* tag, bool is_closure, const grpc_op* ops) {
2441
+ Completion c(BatchSlotForOp(ops[0].op));
2442
+ if (grpc_call_trace.enabled()) {
2443
+ gpr_log(GPR_INFO, "%s[call] StartCompletion %s tag=%p", DebugTag().c_str(),
2444
+ CompletionString(c).c_str(), tag);
2445
+ }
2446
+ if (!is_closure) {
2447
+ grpc_cq_begin_op(cq(), tag);
2448
+ }
2449
+ completion_info_[c.index()].pending = {
2450
+ PendingOpBit(PendingOp::kStartingBatch), is_closure, true, tag};
2451
+ return c;
2452
+ }
2453
+
2454
+ PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
2455
+ const Completion& completion, PendingOp reason) {
2456
+ if (grpc_call_trace.enabled()) {
2457
+ gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
2458
+ CompletionString(completion).c_str(), PendingOpString(reason));
2459
+ }
2460
+ GPR_ASSERT(completion.has_value());
2461
+ auto& pending_op_bits =
2462
+ completion_info_[completion.index()].pending.pending_op_bits;
2463
+ GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0);
2464
+ pending_op_bits |= PendingOpBit(reason);
2465
+ return Completion(completion.index());
2466
+ }
2467
+
2468
+ void PromiseBasedCall::FailCompletion(const Completion& completion,
2469
+ SourceLocation location) {
2470
+ if (grpc_call_trace.enabled()) {
2471
+ gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
2472
+ "%s[call] FailCompletion %s", DebugTag().c_str(),
2473
+ CompletionString(completion).c_str());
2474
+ }
2475
+ completion_info_[completion.index()].pending.success = false;
2476
+ }
2477
+
2478
+ void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
2479
+ PendingOp reason) {
2480
+ if (grpc_call_trace.enabled()) {
2481
+ auto pending_op_bits =
2482
+ completion_info_[completion->index()].pending.pending_op_bits;
2483
+ bool success = completion_info_[completion->index()].pending.success;
2484
+ std::vector<const char*> pending;
2485
+ for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) {
2486
+ if (static_cast<PendingOp>(i) == reason) continue;
2487
+ if (pending_op_bits & (1 << i)) {
2488
+ pending.push_back(PendingOpString(static_cast<PendingOp>(i)));
2489
+ }
2490
+ }
2491
+ gpr_log(
2492
+ GPR_INFO, "%s[call] FinishOpOnCompletion tag:%p %s %s %s",
2493
+ DebugTag().c_str(), completion_info_[completion->index()].pending.tag,
2494
+ CompletionString(*completion).c_str(), PendingOpString(reason),
2495
+ (pending.empty()
2496
+ ? (success ? std::string("done") : std::string("failed"))
2497
+ : absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ",")))
2498
+ .c_str());
2499
+ }
2500
+ const uint8_t i = completion->TakeIndex();
2501
+ GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_));
2502
+ CompletionInfo::Pending& pending = completion_info_[i].pending;
2503
+ GPR_ASSERT(pending.pending_op_bits & PendingOpBit(reason));
2504
+ pending.pending_op_bits &= ~PendingOpBit(reason);
2505
+ auto error = pending.success ? absl::OkStatus() : absl::CancelledError();
2506
+ if (pending.pending_op_bits == 0) {
2507
+ if (pending.is_closure) {
2508
+ ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
2509
+ error);
2510
+ } else {
2511
+ grpc_cq_end_op(
2512
+ cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr,
2513
+ &completion_info_[i].completion);
2514
+ }
2515
+ }
2516
+ }
2517
+
2518
+ void PromiseBasedCall::Update() {
2519
+ keep_polling_ = false;
2520
+ do {
2521
+ UpdateOnce();
2522
+ } while (std::exchange(keep_polling_, false));
2523
+ }
2524
+
2525
+ void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; }
2526
+
2527
+ void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) {
2528
+ MutexLock lock(&mu_);
2529
+ SetCompletionQueueLocked(cq);
2530
+ }
2531
+
2532
+ void PromiseBasedCall::SetCompletionQueueLocked(grpc_completion_queue* cq) {
2533
+ cq_ = cq;
2534
+ GRPC_CQ_INTERNAL_REF(cq, "bind");
2535
+ call_context_.pollent_ =
2536
+ grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
2537
+ }
2538
+
2539
+ void PromiseBasedCall::UpdateDeadline(Timestamp deadline) {
2540
+ if (deadline >= deadline_) return;
2541
+ auto* const event_engine = channel()->event_engine();
2542
+ if (deadline_ != Timestamp::InfFuture()) {
2543
+ if (!event_engine->Cancel(deadline_task_)) return;
2544
+ } else {
2545
+ InternalRef("deadline");
2546
+ }
2547
+ event_engine->RunAfter(deadline - Timestamp::Now(), this);
2548
+ }
2549
+
2550
+ void PromiseBasedCall::ResetDeadline() {
2551
+ if (deadline_ == Timestamp::InfFuture()) return;
2552
+ auto* const event_engine = channel()->event_engine();
2553
+ if (!event_engine->Cancel(deadline_task_)) return;
2554
+ deadline_ = Timestamp::InfFuture();
2555
+ InternalUnref("deadline");
2556
+ }
2557
+
2558
+ void PromiseBasedCall::Run() {
2559
+ ApplicationCallbackExecCtx callback_exec_ctx;
2560
+ ExecCtx exec_ctx;
2561
+ CancelWithError(absl::DeadlineExceededError("Deadline exceeded"));
2562
+ InternalUnref("deadline");
2563
+ }
2564
+
2565
+ void PromiseBasedCall::StartSendMessage(const grpc_op& op,
2566
+ const Completion& completion,
2567
+ PipeSender<MessageHandle>* sender) {
2568
+ GPR_ASSERT(!outstanding_send_.has_value());
2569
+ if (!completed_) {
2570
+ send_message_completion_ =
2571
+ AddOpToCompletion(completion, PendingOp::kSendMessage);
2572
+ SliceBuffer send;
2573
+ grpc_slice_buffer_swap(
2574
+ &op.data.send_message.send_message->data.raw.slice_buffer,
2575
+ send.c_slice_buffer());
2576
+ outstanding_send_.emplace(sender->Push(
2577
+ GetContext<Arena>()->MakePooled<Message>(std::move(send), op.flags)));
2578
+ } else {
2579
+ FailCompletion(completion);
2580
+ }
2581
+ }
2582
+
2583
+ bool PromiseBasedCall::PollSendMessage() {
2584
+ if (!outstanding_send_.has_value()) return true;
2585
+ Poll<bool> r = (*outstanding_send_)();
2586
+ if (const bool* result = r.value_if_ready()) {
2587
+ if (grpc_call_trace.enabled()) {
2588
+ gpr_log(GPR_DEBUG, "%sPollSendMessage completes %s", DebugTag().c_str(),
2589
+ *result ? "successfully" : "with failure");
2590
+ }
2591
+ if (!*result) {
2592
+ FailCompletion(send_message_completion_);
2593
+ return false;
2594
+ }
2595
+ FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
2596
+ outstanding_send_.reset();
2597
+ }
2598
+ return true;
2599
+ }
2600
+
2601
+ void PromiseBasedCall::CancelSendMessage() {
2602
+ if (!outstanding_send_.has_value()) return;
2603
+ FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage);
2604
+ outstanding_send_.reset();
2605
+ }
2606
+
2607
+ void PromiseBasedCall::StartRecvMessage(const grpc_op& op,
2608
+ const Completion& completion,
2609
+ PipeReceiver<MessageHandle>* receiver) {
2610
+ GPR_ASSERT(!outstanding_recv_.has_value());
2611
+ recv_message_ = op.data.recv_message.recv_message;
2612
+ recv_message_completion_ =
2613
+ AddOpToCompletion(completion, PendingOp::kReceiveMessage);
2614
+ outstanding_recv_.emplace(receiver->Next());
2615
+ }
2616
+
2617
+ void PromiseBasedCall::PollRecvMessage(
2618
+ grpc_compression_algorithm incoming_compression_algorithm) {
2619
+ if (!outstanding_recv_.has_value()) return;
2620
+ Poll<NextResult<MessageHandle>> r = (*outstanding_recv_)();
2621
+ if (auto* result = r.value_if_ready()) {
2622
+ outstanding_recv_.reset();
2623
+ if (result->has_value()) {
2624
+ MessageHandle& message = **result;
2625
+ if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
2626
+ (incoming_compression_algorithm != GRPC_COMPRESS_NONE)) {
2627
+ *recv_message_ = grpc_raw_compressed_byte_buffer_create(
2628
+ nullptr, 0, incoming_compression_algorithm);
2629
+ } else {
2630
+ *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
2631
+ }
2632
+ grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
2633
+ &(*recv_message_)->data.raw.slice_buffer);
2634
+ if (grpc_call_trace.enabled()) {
2635
+ gpr_log(GPR_INFO,
2636
+ "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2637
+ "%" PRIdPTR " byte message",
2638
+ DebugTag().c_str(),
2639
+ (*recv_message_)->data.raw.slice_buffer.length);
2640
+ }
2641
+ } else if (result->cancelled()) {
2642
+ if (grpc_call_trace.enabled()) {
2643
+ gpr_log(GPR_INFO,
2644
+ "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2645
+ "end-of-stream with error",
2646
+ DebugTag().c_str());
2647
+ }
2648
+ FailCompletion(recv_message_completion_);
2649
+ *recv_message_ = nullptr;
2650
+ } else {
2651
+ if (grpc_call_trace.enabled()) {
2652
+ gpr_log(GPR_INFO,
2653
+ "%s[call] PollRecvMessage: outstanding_recv finishes: received "
2654
+ "end-of-stream",
2655
+ DebugTag().c_str());
2656
+ }
2657
+ *recv_message_ = nullptr;
2658
+ }
2659
+ FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2660
+ } else if (completed_) {
2661
+ if (grpc_call_trace.enabled()) {
2662
+ gpr_log(GPR_INFO,
2663
+ "%s[call] UpdateOnce: outstanding_recv finishes: promise has "
2664
+ "completed without queuing a message, forcing end-of-stream",
2665
+ DebugTag().c_str());
2666
+ }
2667
+ outstanding_recv_.reset();
2668
+ *recv_message_ = nullptr;
2669
+ FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2670
+ }
2671
+ }
2672
+
2673
+ void PromiseBasedCall::CancelRecvMessage() {
2674
+ if (!outstanding_recv_.has_value()) return;
2675
+ *recv_message_ = nullptr;
2676
+ outstanding_recv_.reset();
2677
+ FinishOpOnCompletion(&recv_message_completion_, PendingOp::kReceiveMessage);
2678
+ }
2679
+
2680
+ ///////////////////////////////////////////////////////////////////////////////
2681
+ // CallContext
2682
+
2683
+ void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
2684
+ call_->RunInContext(std::move(fn));
2685
+ }
2686
+
2687
+ void CallContext::IncrementRefCount(const char* reason) {
2688
+ call_->InternalRef(reason);
2689
+ }
2690
+
2691
+ void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
2692
+
2693
+ gpr_atm* CallContext::peer_string_atm_ptr() {
2694
+ return call_->peer_string_atm_ptr();
2695
+ }
2696
+
2697
+ void CallContext::UpdateDeadline(Timestamp deadline) {
2698
+ call_->UpdateDeadline(deadline);
2699
+ }
2700
+
2701
+ ServerCallContext* CallContext::server_call_context() {
2702
+ return call_->server_call_context();
2703
+ }
2704
+
2705
+ ///////////////////////////////////////////////////////////////////////////////
2706
+ // PublishMetadataArray
2707
+
2708
+ namespace {
2709
+ void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array) {
2710
+ const auto md_count = md->count();
2711
+ if (md_count > array->capacity) {
2712
+ array->capacity =
2713
+ std::max(array->capacity + md->count(), array->capacity * 3 / 2);
2714
+ array->metadata = static_cast<grpc_metadata*>(
2715
+ gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
2716
+ }
2717
+ PublishToAppEncoder encoder(array);
2718
+ md->Encode(&encoder);
2719
+ }
2720
+ } // namespace
2721
+
2722
+ ///////////////////////////////////////////////////////////////////////////////
2723
+ // ClientPromiseBasedCall
2724
+
2725
+ #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
2726
+ class ClientPromiseBasedCall final : public PromiseBasedCall {
2727
+ public:
2728
+ ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
2729
+ : PromiseBasedCall(arena, 1, *args) {
2730
+ global_stats().IncrementClientCallsCreated();
2731
+ ScopedContext context(this);
2732
+ send_initial_metadata_ =
2733
+ GetContext<Arena>()->MakePooled<ClientMetadata>(GetContext<Arena>());
2734
+ send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
2735
+ if (args->authority.has_value()) {
2736
+ send_initial_metadata_->Set(HttpAuthorityMetadata(),
2737
+ std::move(*args->authority));
2738
+ }
2739
+ if (auto* channelz_channel = channel()->channelz_node()) {
2740
+ channelz_channel->RecordCallStarted();
2741
+ }
2742
+ if (args->send_deadline != Timestamp::InfFuture()) {
2743
+ UpdateDeadline(args->send_deadline);
2744
+ }
2745
+ }
2746
+
2747
+ ~ClientPromiseBasedCall() override {
2748
+ ScopedContext context(this);
2749
+ send_initial_metadata_.reset();
2750
+ recv_status_on_client_ = absl::monostate();
2751
+ promise_ = ArenaPromise<ServerMetadataHandle>();
2752
+ // Need to destroy the pipes under the ScopedContext above, so we move them
2753
+ // out here and then allow the destructors to run at end of scope, but
2754
+ // before context.
2755
+ auto c2s = std::move(client_to_server_messages_);
2756
+ auto s2c = std::move(server_to_client_messages_);
2757
+ auto sim = std::move(server_initial_metadata_);
2758
+ }
2759
+
2760
+ absl::string_view GetServerAuthority() const override { abort(); }
2761
+ void CancelWithErrorLocked(grpc_error_handle error) override
2762
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2763
+ bool is_trailers_only() const override {
2764
+ MutexLock lock(mu());
2765
+ return is_trailers_only_;
2766
+ }
2767
+ bool failed_before_recv_message() const override { abort(); }
2768
+
2769
+ grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
2770
+ bool is_notify_tag_closure) override;
2771
+
2772
+ std::string DebugTag() const override {
2773
+ return absl::StrFormat("CLIENT_CALL[%p]: ", this);
2774
+ }
2775
+
2776
+ private:
2777
+ // Poll the underlying promise (and sundry objects) once.
2778
+ void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
2779
+ // Finish the call with the given status/trailing metadata.
2780
+ void Finish(ServerMetadataHandle trailing_metadata)
2781
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2782
+ // Validate that a set of ops is valid for a client call.
2783
+ grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const
2784
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2785
+ // Commit a valid batch of operations to be executed.
2786
+ void CommitBatch(const grpc_op* ops, size_t nops,
2787
+ const Completion& completion)
2788
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2789
+ // Start the underlying promise.
2790
+ void StartPromise(ClientMetadataHandle client_initial_metadata)
2791
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2792
+ // Publish status out to the application.
2793
+ void PublishStatus(
2794
+ grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
2795
+ ServerMetadataHandle trailing_metadata)
2796
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2797
+ // Publish server initial metadata out to the application.
2798
+ void PublishInitialMetadata(ServerMetadata* metadata)
2799
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
2800
+
2801
+ ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
2802
+ Pipe<ServerMetadataHandle> server_initial_metadata_ ABSL_GUARDED_BY(mu()){
2803
+ arena()};
2804
+ Pipe<MessageHandle> client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()};
2805
+ Pipe<MessageHandle> server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()};
2806
+
2807
+ ClientMetadataHandle send_initial_metadata_;
2808
+ grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr;
2809
+ absl::variant<absl::monostate,
2810
+ grpc_op::grpc_op_data::grpc_op_recv_status_on_client,
2811
+ ServerMetadataHandle>
2812
+ recv_status_on_client_ ABSL_GUARDED_BY(mu());
2813
+ absl::optional<PipeReceiverNextType<ServerMetadataHandle>>
2814
+ server_initial_metadata_ready_;
2815
+ absl::optional<grpc_compression_algorithm> incoming_compression_algorithm_;
2816
+ Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu());
2817
+ Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu());
2818
+ Completion close_send_completion_ ABSL_GUARDED_BY(mu());
2819
+ bool is_trailers_only_ ABSL_GUARDED_BY(mu());
2820
+ };
2821
+
2822
+ void ClientPromiseBasedCall::StartPromise(
2823
+ ClientMetadataHandle client_initial_metadata) {
2824
+ GPR_ASSERT(!promise_.has_value());
2825
+ promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{
2826
+ std::move(client_initial_metadata),
2827
+ &server_initial_metadata_.sender,
2828
+ &client_to_server_messages_.receiver,
2829
+ &server_to_client_messages_.sender,
2830
+ });
2831
+ }
2832
+
2833
+ void ClientPromiseBasedCall::CancelWithErrorLocked(grpc_error_handle error) {
2834
+ ScopedContext context(this);
2835
+ Finish(ServerMetadataFromStatus(grpc_error_to_absl_status(error)));
2836
+ }
2837
+
2838
+ grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
2839
+ size_t nops) const {
2840
+ BitSet<8> got_ops;
2841
+ for (size_t op_idx = 0; op_idx < nops; op_idx++) {
2842
+ const grpc_op& op = ops[op_idx];
2843
+ switch (op.op) {
2844
+ case GRPC_OP_SEND_INITIAL_METADATA:
2845
+ if (!AreInitialMetadataFlagsValid(op.flags)) {
2846
+ return GRPC_CALL_ERROR_INVALID_FLAGS;
2847
+ }
2848
+ if (!ValidateMetadata(op.data.send_initial_metadata.count,
2849
+ op.data.send_initial_metadata.metadata)) {
2850
+ return GRPC_CALL_ERROR_INVALID_METADATA;
2851
+ }
2852
+ break;
2853
+ case GRPC_OP_SEND_MESSAGE:
2854
+ if (!AreWriteFlagsValid(op.flags)) {
2855
+ return GRPC_CALL_ERROR_INVALID_FLAGS;
2856
+ }
2857
+ break;
2858
+ case GRPC_OP_RECV_INITIAL_METADATA:
2859
+ case GRPC_OP_RECV_MESSAGE:
2860
+ case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
2861
+ case GRPC_OP_RECV_STATUS_ON_CLIENT:
2862
+ if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
2863
+ break;
2864
+ case GRPC_OP_RECV_CLOSE_ON_SERVER:
2865
+ case GRPC_OP_SEND_STATUS_FROM_SERVER:
2866
+ return GRPC_CALL_ERROR_NOT_ON_CLIENT;
2867
+ }
2868
+ if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
2869
+ got_ops.set(op.op);
2870
+ }
2871
+ return GRPC_CALL_OK;
2872
+ }
2873
+
2874
+ void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
2875
+ const Completion& completion) {
2876
+ for (size_t op_idx = 0; op_idx < nops; op_idx++) {
2877
+ const grpc_op& op = ops[op_idx];
2878
+ switch (op.op) {
2879
+ case GRPC_OP_SEND_INITIAL_METADATA: {
2880
+ // compression not implemented
2881
+ GPR_ASSERT(
2882
+ !op.data.send_initial_metadata.maybe_compression_level.is_set);
2883
+ if (!completed()) {
2884
+ CToMetadata(op.data.send_initial_metadata.metadata,
2885
+ op.data.send_initial_metadata.count,
2886
+ send_initial_metadata_.get());
2887
+ StartPromise(std::move(send_initial_metadata_));
2888
+ }
2889
+ } break;
2890
+ case GRPC_OP_RECV_INITIAL_METADATA: {
2891
+ recv_initial_metadata_ =
2892
+ op.data.recv_initial_metadata.recv_initial_metadata;
2893
+ server_initial_metadata_ready_.emplace(
2894
+ server_initial_metadata_.receiver.Next());
2895
+ recv_initial_metadata_completion_ =
2896
+ AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata);
2897
+ } break;
2898
+ case GRPC_OP_RECV_STATUS_ON_CLIENT: {
2899
+ recv_status_on_client_completion_ =
2900
+ AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient);
2901
+ if (auto* finished_metadata =
2902
+ absl::get_if<ServerMetadataHandle>(&recv_status_on_client_)) {
2903
+ PublishStatus(op.data.recv_status_on_client,
2904
+ std::move(*finished_metadata));
2905
+ } else {
2906
+ recv_status_on_client_ = op.data.recv_status_on_client;
2907
+ }
2908
+ } break;
2909
+ case GRPC_OP_SEND_MESSAGE:
2910
+ StartSendMessage(op, completion, &client_to_server_messages_.sender);
2911
+ break;
2912
+ case GRPC_OP_RECV_MESSAGE:
2913
+ StartRecvMessage(op, completion, &server_to_client_messages_.receiver);
2914
+ break;
2915
+ case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
2916
+ close_send_completion_ =
2917
+ AddOpToCompletion(completion, PendingOp::kSendCloseFromClient);
2918
+ GPR_ASSERT(close_send_completion_.has_value());
2919
+ break;
2920
+ case GRPC_OP_SEND_STATUS_FROM_SERVER:
2921
+ case GRPC_OP_RECV_CLOSE_ON_SERVER:
2922
+ abort(); // unreachable
2923
+ }
2924
+ }
2925
+ }
2926
+
2927
+ grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
2928
+ size_t nops,
2929
+ void* notify_tag,
2930
+ bool is_notify_tag_closure) {
2931
+ MutexLock lock(mu());
2932
+ ScopedContext activity_context(this);
2933
+ if (nops == 0) {
2934
+ EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
2935
+ return GRPC_CALL_OK;
2936
+ }
2937
+ const grpc_call_error validation_result = ValidateBatch(ops, nops);
2938
+ if (validation_result != GRPC_CALL_OK) {
2939
+ return validation_result;
2940
+ }
2941
+ Completion completion =
2942
+ StartCompletion(notify_tag, is_notify_tag_closure, ops);
2943
+ CommitBatch(ops, nops, completion);
2944
+ Update();
2945
+ FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
2946
+ return GRPC_CALL_OK;
2947
+ }
2948
+
2949
+ void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) {
2950
+ incoming_compression_algorithm_ =
2951
+ metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
2952
+ server_initial_metadata_ready_.reset();
2953
+ GPR_ASSERT(recv_initial_metadata_ != nullptr);
2954
+ PublishMetadataArray(metadata,
2955
+ std::exchange(recv_initial_metadata_, nullptr));
2956
+ FinishOpOnCompletion(&recv_initial_metadata_completion_,
2957
+ PendingOp::kReceiveInitialMetadata);
2958
+ }
2959
+
2960
+ void ClientPromiseBasedCall::UpdateOnce() {
2961
+ if (grpc_call_trace.enabled()) {
2962
+ gpr_log(GPR_INFO, "%s[call] UpdateOnce: %s%shas_promise=%s",
2963
+ DebugTag().c_str(),
2964
+ PresentAndCompletionText("server_initial_metadata_ready",
2965
+ server_initial_metadata_ready_.has_value(),
2966
+ recv_initial_metadata_completion_)
2967
+ .c_str(),
2968
+ PollStateDebugString().c_str(),
2969
+ promise_.has_value() ? "true" : "false");
2970
+ }
2971
+ if (server_initial_metadata_ready_.has_value()) {
2972
+ Poll<NextResult<ServerMetadataHandle>> r =
2973
+ (*server_initial_metadata_ready_)();
2974
+ if (auto* server_initial_metadata = r.value_if_ready()) {
2975
+ PublishInitialMetadata(server_initial_metadata->value().get());
2976
+ } else if (completed()) {
2977
+ ServerMetadata no_metadata{GetContext<Arena>()};
2978
+ PublishInitialMetadata(&no_metadata);
2979
+ }
2980
+ }
2981
+ if (!PollSendMessage()) {
2982
+ Finish(ServerMetadataFromStatus(absl::Status(
2983
+ absl::StatusCode::kInternal, "Failed to send message to server")));
2984
+ }
2985
+ if (!is_sending() && close_send_completion_.has_value()) {
2986
+ client_to_server_messages_.sender.Close();
2987
+ FinishOpOnCompletion(&close_send_completion_,
2988
+ PendingOp::kSendCloseFromClient);
2989
+ }
2990
+ if (promise_.has_value()) {
2991
+ Poll<ServerMetadataHandle> r = promise_();
2992
+ if (grpc_call_trace.enabled()) {
2993
+ gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
2994
+ DebugTag().c_str(),
2995
+ PollToString(r, [](const ServerMetadataHandle& h) {
2996
+ return h->DebugString();
2997
+ }).c_str());
2998
+ }
2999
+ if (auto* result = r.value_if_ready()) {
3000
+ AcceptTransportStatsFromContext();
3001
+ Finish(std::move(*result));
3002
+ }
3003
+ }
3004
+ if (incoming_compression_algorithm_.has_value()) {
3005
+ PollRecvMessage(*incoming_compression_algorithm_);
3006
+ }
3007
+ }
3008
+
3009
+ void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
3010
+ if (grpc_call_trace.enabled()) {
3011
+ gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
3012
+ trailing_metadata->DebugString().c_str());
3013
+ }
3014
+ promise_ = ArenaPromise<ServerMetadataHandle>();
3015
+ ResetDeadline();
3016
+ set_completed();
3017
+ if (recv_initial_metadata_ != nullptr) {
3018
+ ForceImmediateRepoll();
3019
+ }
3020
+ const bool pending_initial_metadata =
3021
+ server_initial_metadata_ready_.has_value();
3022
+ if (!pending_initial_metadata) {
3023
+ server_initial_metadata_ready_.emplace(
3024
+ server_initial_metadata_.receiver.Next());
3025
+ }
3026
+ Poll<NextResult<ServerMetadataHandle>> r =
3027
+ (*server_initial_metadata_ready_)();
3028
+ server_initial_metadata_ready_.reset();
3029
+ if (auto* result = r.value_if_ready()) {
3030
+ if (pending_initial_metadata) PublishInitialMetadata(result->value().get());
3031
+ is_trailers_only_ = false;
3032
+ } else {
3033
+ if (pending_initial_metadata) {
3034
+ ServerMetadata no_metadata{GetContext<Arena>()};
3035
+ PublishInitialMetadata(&no_metadata);
3036
+ }
3037
+ is_trailers_only_ = true;
3038
+ }
3039
+ if (auto* channelz_channel = channel()->channelz_node()) {
3040
+ if (trailing_metadata->get(GrpcStatusMetadata())
3041
+ .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
3042
+ channelz_channel->RecordCallSucceeded();
3043
+ } else {
3044
+ channelz_channel->RecordCallFailed();
3045
+ }
3046
+ }
3047
+ if (auto* status_request =
3048
+ absl::get_if<grpc_op::grpc_op_data::grpc_op_recv_status_on_client>(
3049
+ &recv_status_on_client_)) {
3050
+ PublishStatus(*status_request, std::move(trailing_metadata));
3051
+ } else {
3052
+ recv_status_on_client_ = std::move(trailing_metadata);
3053
+ }
3054
+ }
3055
+
3056
+ namespace {
3057
+ std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
3058
+ std::string out = absl::StrCat(
3059
+ trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
3060
+ ? "Error received from peer"
3061
+ : "Error generated by client",
3062
+ "grpc_status: ",
3063
+ grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
3064
+ .value_or(GRPC_STATUS_UNKNOWN)));
3065
+ if (const Slice* message =
3066
+ trailing_metadata->get_pointer(GrpcMessageMetadata())) {
3067
+ absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
3068
+ }
3069
+ if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
3070
+ absl::StrAppend(&out, "\nStatus Context:");
3071
+ for (const std::string& annotation : *annotations) {
3072
+ absl::StrAppend(&out, "\n ", annotation);
3073
+ }
3074
+ }
3075
+ return out;
3076
+ }
3077
+ } // namespace
3078
+
3079
+ void ClientPromiseBasedCall::PublishStatus(
3080
+ grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
3081
+ ServerMetadataHandle trailing_metadata) {
3082
+ const grpc_status_code status = trailing_metadata->get(GrpcStatusMetadata())
3083
+ .value_or(GRPC_STATUS_UNKNOWN);
3084
+ *op_args.status = status;
3085
+ absl::string_view message_string;
3086
+ if (Slice* message = trailing_metadata->get_pointer(GrpcMessageMetadata())) {
3087
+ message_string = message->as_string_view();
3088
+ *op_args.status_details = message->Ref().TakeCSlice();
3089
+ } else {
3090
+ *op_args.status_details = grpc_empty_slice();
3091
+ }
3092
+ if (message_string.empty()) {
3093
+ RunFinalization(status, nullptr);
3094
+ } else {
3095
+ std::string error_string(message_string);
3096
+ RunFinalization(status, error_string.c_str());
3097
+ }
3098
+ if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
3099
+ *op_args.error_string =
3100
+ gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
3101
+ }
3102
+ PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata);
3103
+ // Clear state saying we have a RECV_STATUS_ON_CLIENT outstanding
3104
+ // (so we don't call through twice)
3105
+ recv_status_on_client_ = absl::monostate();
3106
+ FinishOpOnCompletion(&recv_status_on_client_completion_,
3107
+ PendingOp::kReceiveStatusOnClient);
3108
+ }
3109
+ #endif
3110
+
3111
+ ///////////////////////////////////////////////////////////////////////////////
3112
+ // ServerPromiseBasedCall
3113
+
3114
+ #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
3115
+
3116
+ class ServerPromiseBasedCall final : public PromiseBasedCall {
3117
+ public:
3118
+ ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
3119
+
3120
+ void CancelWithErrorLocked(grpc_error_handle) override
3121
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3122
+ grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
3123
+ bool is_notify_tag_closure) override;
3124
+ bool failed_before_recv_message() const override { abort(); }
3125
+ bool is_trailers_only() const override { abort(); }
3126
+ absl::string_view GetServerAuthority() const override { return ""; }
3127
+
3128
+ // Polling order for the server promise stack:
3129
+ //
3130
+ // │ ┌───────────────────────────────────────┐
3131
+ // │ │ ServerPromiseBasedCall::UpdateOnce ├──► Lifetime management,
3132
+ // │ ├───────────────────────────────────────┤ signal call end to app
3133
+ // │ │ ConnectedChannel ├─┐
3134
+ // │ ├───────────────────────────────────────┤ └► Interactions with the
3135
+ // │ │ ... closest to transport filter │ transport - send/recv msgs
3136
+ // │ ├───────────────────────────────────────┤ and metadata, call phase
3137
+ // │ │ ... │ ordering
3138
+ // │ ├───────────────────────────────────────┤
3139
+ // │ │ ... closest to app filter │ ┌► Request matching, initial
3140
+ // │ ├───────────────────────────────────────┤ │ setup, publishing call to
3141
+ // │ │ Server::ChannelData::MakeCallPromise ├─┘ application
3142
+ // │ ├───────────────────────────────────────┤
3143
+ // │ │ ServerPromiseBasedCall::PollTopOfCall ├──► Application interactions,
3144
+ // ▼ └───────────────────────────────────────┘ forwarding messages,
3145
+ // Polling & sending trailing metadata
3146
+ // instantiation
3147
+ // order
3148
+
3149
+ void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override;
3150
+ Poll<ServerMetadataHandle> PollTopOfCall()
3151
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3152
+
3153
+ std::string DebugTag() const override {
3154
+ return absl::StrFormat("SERVER_CALL[%p]: ", this);
3155
+ }
3156
+
3157
+ ServerCallContext* server_call_context() override { return &call_context_; }
3158
+
3159
+ private:
3160
+ class RecvCloseOpCancelState {
3161
+ public:
3162
+ // Request that receiver be filled in per grpc_op_recv_close_on_server.
3163
+ // Returns true if the request can be fulfilled immediately.
3164
+ // Returns false if the request will be fulfilled later.
3165
+ bool ReceiveCloseOnServerOpStarted(int* receiver) {
3166
+ switch (state_) {
3167
+ case kUnset:
3168
+ state_ = reinterpret_cast<uintptr_t>(receiver);
3169
+ return false;
3170
+ case kFinishedWithFailure:
3171
+ *receiver = 1;
3172
+ return true;
3173
+ case kFinishedWithSuccess:
3174
+ *receiver = 0;
3175
+ return true;
3176
+ default:
3177
+ abort(); // unreachable
3178
+ }
3179
+ }
3180
+
3181
+ // Mark the call as having completed.
3182
+ // Returns true if this finishes a previous RequestReceiveCloseOnServer.
3183
+ bool CompleteCall(bool success) {
3184
+ switch (state_) {
3185
+ case kUnset:
3186
+ state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
3187
+ return false;
3188
+ case kFinishedWithFailure:
3189
+ case kFinishedWithSuccess:
3190
+ abort(); // unreachable
3191
+ default:
3192
+ *reinterpret_cast<int*>(state_) = success ? 0 : 1;
3193
+ state_ = success ? kFinishedWithSuccess : kFinishedWithFailure;
3194
+ return true;
3195
+ }
3196
+ }
3197
+
3198
+ std::string ToString() const {
3199
+ switch (state_) {
3200
+ case kUnset:
3201
+ return "Unset";
3202
+ case kFinishedWithFailure:
3203
+ return "FinishedWithFailure";
3204
+ case kFinishedWithSuccess:
3205
+ return "FinishedWithSuccess";
3206
+ default:
3207
+ return absl::StrFormat("WaitingForReceiver(%p)",
3208
+ reinterpret_cast<void*>(state_));
3209
+ }
3210
+ }
3211
+
3212
+ private:
3213
+ static constexpr uintptr_t kUnset = 0;
3214
+ static constexpr uintptr_t kFinishedWithFailure = 1;
3215
+ static constexpr uintptr_t kFinishedWithSuccess = 2;
3216
+ // Holds one of kUnset, kFinishedWithFailure, or kFinishedWithSuccess
3217
+ // OR an int* that wants to receive the final status.
3218
+ uintptr_t state_ = kUnset;
3219
+ };
3220
+
3221
+ grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
3222
+ void CommitBatch(const grpc_op* ops, size_t nops,
3223
+ const Completion& completion)
3224
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu());
3225
+
3226
+ friend class ServerCallContext;
3227
+ ServerCallContext call_context_;
3228
+ Server* const server_;
3229
+ ArenaPromise<ServerMetadataHandle> promise_ ABSL_GUARDED_BY(mu());
3230
+ PipeSender<MessageHandle>* server_to_client_messages_ ABSL_GUARDED_BY(mu()) =
3231
+ nullptr;
3232
+ PipeReceiver<MessageHandle>* client_to_server_messages_
3233
+ ABSL_GUARDED_BY(mu()) = nullptr;
3234
+ using SendInitialMetadataState =
3235
+ absl::variant<absl::monostate, PipeSender<ServerMetadataHandle>*,
3236
+ typename PipeSender<ServerMetadataHandle>::PushType>;
3237
+ SendInitialMetadataState send_initial_metadata_state_ ABSL_GUARDED_BY(mu()) =
3238
+ absl::monostate{};
3239
+ ServerMetadataHandle send_trailing_metadata_ ABSL_GUARDED_BY(mu());
3240
+ grpc_compression_algorithm incoming_compression_algorithm_
3241
+ ABSL_GUARDED_BY(mu());
3242
+ RecvCloseOpCancelState recv_close_op_cancel_state_ ABSL_GUARDED_BY(mu());
3243
+ Completion recv_close_completion_ ABSL_GUARDED_BY(mu());
3244
+ bool cancel_send_and_receive_ ABSL_GUARDED_BY(mu()) = false;
3245
+ Completion send_status_from_server_completion_ ABSL_GUARDED_BY(mu());
3246
+ ClientMetadataHandle client_initial_metadata_ ABSL_GUARDED_BY(mu());
3247
+ };
3248
+
3249
+ ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
3250
+ grpc_call_create_args* args)
3251
+ : PromiseBasedCall(arena, 0, *args),
3252
+ call_context_(this, args->server_transport_data),
3253
+ server_(args->server) {
3254
+ global_stats().IncrementServerCallsCreated();
3255
+ channelz::ServerNode* channelz_node = server_->channelz_node();
3256
+ if (channelz_node != nullptr) {
3257
+ channelz_node->RecordCallStarted();
3258
+ }
3259
+ MutexLock lock(mu());
3260
+ ScopedContext activity_context(this);
3261
+ promise_ = channel()->channel_stack()->MakeServerCallPromise(
3262
+ CallArgs{nullptr, nullptr, nullptr, nullptr});
3263
+ }
3264
+
3265
+ Poll<ServerMetadataHandle> ServerPromiseBasedCall::PollTopOfCall() {
3266
+ if (grpc_call_trace.enabled()) {
3267
+ gpr_log(GPR_INFO, "%s[call] PollTopOfCall: %s%s%s", DebugTag().c_str(),
3268
+ cancel_send_and_receive_ ? "force-" : "",
3269
+ send_trailing_metadata_ != nullptr
3270
+ ? absl::StrCat("send-metadata:",
3271
+ send_trailing_metadata_->DebugString(), " ")
3272
+ .c_str()
3273
+ : " ",
3274
+ PollStateDebugString().c_str());
3275
+ }
3276
+
3277
+ if (cancel_send_and_receive_) {
3278
+ CancelSendMessage();
3279
+ CancelRecvMessage();
3280
+ }
3281
+
3282
+ PollSendMessage();
3283
+ PollRecvMessage(incoming_compression_algorithm_);
3284
+
3285
+ if (!is_sending() && send_trailing_metadata_ != nullptr) {
3286
+ server_to_client_messages_->Close();
3287
+ return std::move(send_trailing_metadata_);
3288
+ }
3289
+
3290
+ return Pending{};
3291
+ }
3292
+
3293
+ void ServerPromiseBasedCall::UpdateOnce() {
3294
+ if (grpc_call_trace.enabled()) {
3295
+ gpr_log(
3296
+ GPR_INFO, "%s[call] UpdateOnce: recv_close:%s%s %s%shas_promise=%s",
3297
+ DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
3298
+ recv_close_completion_.has_value()
3299
+ ? absl::StrCat(":", CompletionString(recv_close_completion_))
3300
+ .c_str()
3301
+ : "",
3302
+ send_status_from_server_completion_.has_value()
3303
+ ? absl::StrCat(
3304
+ "send_status:",
3305
+ CompletionString(send_status_from_server_completion_), " ")
3306
+ .c_str()
3307
+ : "",
3308
+ PollStateDebugString().c_str(),
3309
+ promise_.has_value() ? "true" : "false");
3310
+ }
3311
+ if (auto* p =
3312
+ absl::get_if<typename PipeSender<ServerMetadataHandle>::PushType>(
3313
+ &send_initial_metadata_state_)) {
3314
+ if ((*p)().ready()) {
3315
+ send_initial_metadata_state_ = absl::monostate{};
3316
+ }
3317
+ }
3318
+ if (promise_.has_value()) {
3319
+ auto r = promise_();
3320
+ if (grpc_call_trace.enabled()) {
3321
+ gpr_log(GPR_INFO, "%s[call] UpdateOnce: promise returns %s",
3322
+ DebugTag().c_str(),
3323
+ PollToString(r, [](const ServerMetadataHandle& h) {
3324
+ return h->DebugString();
3325
+ }).c_str());
3326
+ }
3327
+ if (auto* result = r.value_if_ready()) {
3328
+ if (grpc_call_trace.enabled()) {
3329
+ gpr_log(GPR_INFO, "%s[call] UpdateOnce: GotResult %s result:%s",
3330
+ DebugTag().c_str(),
3331
+ recv_close_op_cancel_state_.ToString().c_str(),
3332
+ (*result)->DebugString().c_str());
3333
+ }
3334
+ if (recv_close_op_cancel_state_.CompleteCall(
3335
+ (*result)->get(GrpcStatusFromWire()).value_or(false))) {
3336
+ FinishOpOnCompletion(&recv_close_completion_,
3337
+ PendingOp::kReceiveCloseOnServer);
3338
+ }
3339
+ channelz::ServerNode* channelz_node = server_->channelz_node();
3340
+ if (channelz_node != nullptr) {
3341
+ if ((*result)
3342
+ ->get(GrpcStatusMetadata())
3343
+ .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
3344
+ channelz_node->RecordCallSucceeded();
3345
+ } else {
3346
+ channelz_node->RecordCallFailed();
3347
+ }
3348
+ }
3349
+ if (send_status_from_server_completion_.has_value()) {
3350
+ FinishOpOnCompletion(&send_status_from_server_completion_,
3351
+ PendingOp::kSendStatusFromServer);
3352
+ }
3353
+ CancelSendMessage();
3354
+ CancelRecvMessage();
3355
+ set_completed();
3356
+ promise_ = ArenaPromise<ServerMetadataHandle>();
3357
+ }
3358
+ }
3359
+ }
3360
+
3361
+ grpc_call_error ServerPromiseBasedCall::ValidateBatch(const grpc_op* ops,
3362
+ size_t nops) const {
3363
+ BitSet<8> got_ops;
3364
+ for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3365
+ const grpc_op& op = ops[op_idx];
3366
+ switch (op.op) {
3367
+ case GRPC_OP_SEND_INITIAL_METADATA:
3368
+ if (!AreInitialMetadataFlagsValid(op.flags)) {
3369
+ return GRPC_CALL_ERROR_INVALID_FLAGS;
3370
+ }
3371
+ if (!ValidateMetadata(op.data.send_initial_metadata.count,
3372
+ op.data.send_initial_metadata.metadata)) {
3373
+ return GRPC_CALL_ERROR_INVALID_METADATA;
3374
+ }
3375
+ break;
3376
+ case GRPC_OP_SEND_MESSAGE:
3377
+ if (!AreWriteFlagsValid(op.flags)) {
3378
+ return GRPC_CALL_ERROR_INVALID_FLAGS;
3379
+ }
3380
+ break;
3381
+ case GRPC_OP_RECV_MESSAGE:
3382
+ case GRPC_OP_RECV_CLOSE_ON_SERVER:
3383
+ case GRPC_OP_SEND_STATUS_FROM_SERVER:
3384
+ if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
3385
+ break;
3386
+ case GRPC_OP_RECV_INITIAL_METADATA:
3387
+ case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
3388
+ case GRPC_OP_RECV_STATUS_ON_CLIENT:
3389
+ return GRPC_CALL_ERROR_NOT_ON_SERVER;
3390
+ }
3391
+ if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
3392
+ got_ops.set(op.op);
3393
+ }
3394
+ return GRPC_CALL_OK;
3395
+ }
3396
+
3397
+ void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
3398
+ const Completion& completion) {
3399
+ for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3400
+ const grpc_op& op = ops[op_idx];
3401
+ switch (op.op) {
3402
+ case GRPC_OP_SEND_INITIAL_METADATA: {
3403
+ // compression not implemented
3404
+ GPR_ASSERT(
3405
+ !op.data.send_initial_metadata.maybe_compression_level.is_set);
3406
+ if (!completed()) {
3407
+ auto metadata = arena()->MakePooled<ServerMetadata>(arena());
3408
+ CToMetadata(op.data.send_initial_metadata.metadata,
3409
+ op.data.send_initial_metadata.count, metadata.get());
3410
+ if (grpc_call_trace.enabled()) {
3411
+ gpr_log(GPR_INFO, "%s[call] Send initial metadata",
3412
+ DebugTag().c_str());
3413
+ }
3414
+ auto* pipe = absl::get<PipeSender<ServerMetadataHandle>*>(
3415
+ send_initial_metadata_state_);
3416
+ send_initial_metadata_state_ = pipe->Push(std::move(metadata));
3417
+ }
3418
+ } break;
3419
+ case GRPC_OP_SEND_MESSAGE:
3420
+ StartSendMessage(op, completion, server_to_client_messages_);
3421
+ break;
3422
+ case GRPC_OP_RECV_MESSAGE:
3423
+ StartRecvMessage(op, completion, client_to_server_messages_);
3424
+ break;
3425
+ case GRPC_OP_SEND_STATUS_FROM_SERVER:
3426
+ send_trailing_metadata_ = arena()->MakePooled<ServerMetadata>(arena());
3427
+ CToMetadata(op.data.send_status_from_server.trailing_metadata,
3428
+ op.data.send_status_from_server.trailing_metadata_count,
3429
+ send_trailing_metadata_.get());
3430
+ send_trailing_metadata_->Set(GrpcStatusMetadata(),
3431
+ op.data.send_status_from_server.status);
3432
+ if (auto* details = op.data.send_status_from_server.status_details) {
3433
+ send_trailing_metadata_->Set(GrpcMessageMetadata(),
3434
+ Slice(CSliceRef(*details)));
3435
+ }
3436
+ send_status_from_server_completion_ =
3437
+ AddOpToCompletion(completion, PendingOp::kSendStatusFromServer);
3438
+ break;
3439
+ case GRPC_OP_RECV_CLOSE_ON_SERVER:
3440
+ if (grpc_call_trace.enabled()) {
3441
+ gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
3442
+ DebugTag().c_str(),
3443
+ recv_close_op_cancel_state_.ToString().c_str());
3444
+ }
3445
+ if (!recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
3446
+ op.data.recv_close_on_server.cancelled)) {
3447
+ recv_close_completion_ =
3448
+ AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
3449
+ }
3450
+ break;
3451
+ case GRPC_OP_RECV_STATUS_ON_CLIENT:
3452
+ case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
3453
+ case GRPC_OP_RECV_INITIAL_METADATA:
3454
+ abort(); // unreachable
3455
+ }
3456
+ }
3457
+ }
3458
+
3459
+ grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
3460
+ size_t nops,
3461
+ void* notify_tag,
3462
+ bool is_notify_tag_closure) {
3463
+ MutexLock lock(mu());
3464
+ ScopedContext activity_context(this);
3465
+ if (nops == 0) {
3466
+ EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
3467
+ return GRPC_CALL_OK;
3468
+ }
3469
+ const grpc_call_error validation_result = ValidateBatch(ops, nops);
3470
+ if (validation_result != GRPC_CALL_OK) {
3471
+ return validation_result;
3472
+ }
3473
+ Completion completion =
3474
+ StartCompletion(notify_tag, is_notify_tag_closure, ops);
3475
+ CommitBatch(ops, nops, completion);
3476
+ Update();
3477
+ FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
3478
+ return GRPC_CALL_OK;
3479
+ }
3480
+
3481
+ void ServerPromiseBasedCall::CancelWithErrorLocked(absl::Status error) {
3482
+ if (!promise_.has_value()) return;
3483
+ cancel_send_and_receive_ = true;
3484
+ send_trailing_metadata_ = ServerMetadataFromStatus(error, arena());
3485
+ ForceWakeup();
3486
+ }
3487
+
3488
+ #endif
3489
+
3490
+ #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
3491
+ ArenaPromise<ServerMetadataHandle>
3492
+ ServerCallContext::MakeTopOfServerCallPromise(
3493
+ CallArgs call_args, grpc_completion_queue* cq,
3494
+ grpc_metadata_array* publish_initial_metadata,
3495
+ absl::FunctionRef<void(grpc_call* call)> publish) {
3496
+ call_->mu()->AssertHeld();
3497
+ call_->SetCompletionQueueLocked(cq);
3498
+ call_->server_to_client_messages_ = call_args.server_to_client_messages;
3499
+ call_->client_to_server_messages_ = call_args.client_to_server_messages;
3500
+ call_->send_initial_metadata_state_ = call_args.server_initial_metadata;
3501
+ call_->incoming_compression_algorithm_ =
3502
+ call_args.client_initial_metadata->get(GrpcEncodingMetadata())
3503
+ .value_or(GRPC_COMPRESS_NONE);
3504
+ call_->client_initial_metadata_ =
3505
+ std::move(call_args.client_initial_metadata);
3506
+ PublishMetadataArray(call_->client_initial_metadata_.get(),
3507
+ publish_initial_metadata);
3508
+ call_->ExternalRef();
3509
+ publish(call_->c_ptr());
3510
+ return [this]() {
3511
+ call_->mu()->AssertHeld();
3512
+ return call_->PollTopOfCall();
3513
+ };
3514
+ }
3515
+ #else
3516
+ ArenaPromise<ServerMetadataHandle>
3517
+ ServerCallContext::MakeTopOfServerCallPromise(
3518
+ CallArgs, grpc_completion_queue*, grpc_metadata_array*,
3519
+ absl::FunctionRef<void(grpc_call*)>) {
3520
+ (void)call_;
3521
+ Crash("Promise-based server call is not enabled");
3522
+ }
3523
+ #endif
3524
+
1798
3525
  } // namespace grpc_core
1799
3526
 
3527
+ ///////////////////////////////////////////////////////////////////////////////
3528
+ // C-based API
3529
+
1800
3530
  void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
1801
3531
  grpc_core::ExecCtx exec_ctx;
1802
3532
  return grpc_core::Call::FromC(call)->arena()->Alloc(size);
@@ -1808,6 +3538,20 @@ size_t grpc_call_get_initial_size_estimate() {
1808
3538
 
1809
3539
  grpc_error_handle grpc_call_create(grpc_call_create_args* args,
1810
3540
  grpc_call** out_call) {
3541
+ #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
3542
+ if (grpc_core::IsPromiseBasedClientCallEnabled() &&
3543
+ args->server_transport_data == nullptr && args->channel->is_promising()) {
3544
+ return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
3545
+ args, out_call);
3546
+ }
3547
+ #endif
3548
+ #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
3549
+ if (grpc_core::IsPromiseBasedServerCallEnabled() &&
3550
+ args->server_transport_data != nullptr && args->channel->is_promising()) {
3551
+ return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
3552
+ args, out_call);
3553
+ }
3554
+ #endif
1811
3555
  return grpc_core::FilterStackCall::Create(args, out_call);
1812
3556
  }
1813
3557
 
@@ -1819,6 +3563,7 @@ void grpc_call_set_completion_queue(grpc_call* call,
1819
3563
  void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
1820
3564
 
1821
3565
  void grpc_call_unref(grpc_call* c) {
3566
+ grpc_core::ExecCtx exec_ctx;
1822
3567
  grpc_core::Call::FromC(c)->ExternalUnref();
1823
3568
  }
1824
3569
 
@@ -1835,7 +3580,7 @@ grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
1835
3580
  GPR_ASSERT(reserved == nullptr);
1836
3581
  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
1837
3582
  grpc_core::ExecCtx exec_ctx;
1838
- grpc_core::Call::FromC(call)->CancelWithError(GRPC_ERROR_CANCELLED);
3583
+ grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
1839
3584
  return GRPC_CALL_OK;
1840
3585
  }
1841
3586
 
@@ -1855,7 +3600,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call* c,
1855
3600
  }
1856
3601
 
1857
3602
  void grpc_call_cancel_internal(grpc_call* call) {
1858
- grpc_core::Call::FromC(call)->CancelWithError(GRPC_ERROR_CANCELLED);
3603
+ grpc_core::Call::FromC(call)->CancelWithError(absl::CancelledError());
1859
3604
  }
1860
3605
 
1861
3606
  grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
@@ -1928,6 +3673,10 @@ int grpc_call_failed_before_recv_message(const grpc_call* c) {
1928
3673
  return grpc_core::Call::FromC(c)->failed_before_recv_message();
1929
3674
  }
1930
3675
 
3676
+ absl::string_view grpc_call_server_authority(const grpc_call* call) {
3677
+ return grpc_core::Call::FromC(call)->GetServerAuthority();
3678
+ }
3679
+
1931
3680
  const char* grpc_call_error_to_string(grpc_call_error error) {
1932
3681
  switch (error) {
1933
3682
  case GRPC_CALL_ERROR: