grpc 1.45.0 → 1.47.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of grpc might be problematic. Click here for more details.

Files changed (859) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +69 -110
  3. data/include/grpc/event_engine/event_engine.h +42 -7
  4. data/include/grpc/event_engine/memory_allocator.h +0 -15
  5. data/include/grpc/event_engine/port.h +1 -1
  6. data/include/grpc/event_engine/slice.h +286 -0
  7. data/include/grpc/event_engine/slice_buffer.h +112 -0
  8. data/include/grpc/grpc_security.h +11 -0
  9. data/include/grpc/impl/codegen/grpc_types.h +7 -13
  10. data/include/grpc/impl/codegen/port_platform.h +100 -36
  11. data/include/grpc/impl/codegen/slice.h +1 -1
  12. data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +297 -0
  13. data/src/core/ext/filters/channel_idle/channel_idle_filter.h +125 -0
  14. data/src/core/ext/filters/{client_idle → channel_idle}/idle_filter_state.cc +1 -1
  15. data/src/core/ext/filters/{client_idle → channel_idle}/idle_filter_state.h +3 -3
  16. data/src/core/ext/filters/client_channel/backend_metric.cc +17 -12
  17. data/src/core/ext/filters/client_channel/backend_metric.h +19 -9
  18. data/src/core/ext/filters/client_channel/backup_poller.cc +5 -5
  19. data/src/core/ext/filters/client_channel/backup_poller.h +2 -4
  20. data/src/core/ext/filters/client_channel/channel_connectivity.cc +42 -20
  21. data/src/core/ext/filters/client_channel/client_channel.cc +223 -113
  22. data/src/core/ext/filters/client_channel/client_channel.h +28 -6
  23. data/src/core/ext/filters/client_channel/client_channel_channelz.cc +3 -5
  24. data/src/core/ext/filters/client_channel/client_channel_channelz.h +12 -2
  25. data/src/core/ext/filters/client_channel/client_channel_factory.cc +5 -0
  26. data/src/core/ext/filters/client_channel/client_channel_factory.h +8 -0
  27. data/src/core/ext/filters/client_channel/client_channel_plugin.cc +4 -13
  28. data/src/core/ext/filters/client_channel/config_selector.cc +1 -0
  29. data/src/core/ext/filters/client_channel/config_selector.h +7 -5
  30. data/src/core/ext/filters/client_channel/connector.h +8 -2
  31. data/src/core/ext/filters/client_channel/dynamic_filters.cc +13 -2
  32. data/src/core/ext/filters/client_channel/dynamic_filters.h +11 -1
  33. data/src/core/ext/filters/client_channel/global_subchannel_pool.cc +2 -0
  34. data/src/core/ext/filters/client_channel/global_subchannel_pool.h +3 -0
  35. data/src/core/ext/filters/client_channel/health/health_check_client.cc +138 -582
  36. data/src/core/ext/filters/client_channel/health/health_check_client.h +26 -160
  37. data/src/core/ext/filters/client_channel/http_proxy.cc +98 -112
  38. data/src/core/ext/filters/client_channel/http_proxy.h +20 -0
  39. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc +6 -2
  40. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.h +1 -0
  41. data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +49 -0
  42. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc +12 -0
  43. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h +5 -0
  44. data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +9 -3
  45. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +59 -19
  46. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc +4 -0
  47. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +6 -0
  48. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc +10 -1
  49. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +6 -3
  50. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc +408 -0
  51. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h +57 -0
  52. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc +1038 -0
  53. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h +54 -0
  54. data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +60 -52
  55. data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc +277 -195
  56. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +158 -101
  57. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h +2 -0
  58. data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +46 -20
  59. data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +182 -171
  60. data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +39 -36
  61. data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +117 -63
  62. data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +142 -63
  63. data/src/core/ext/filters/client_channel/lb_policy/xds/xds.h +6 -0
  64. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +30 -4
  65. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +37 -10
  66. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +300 -291
  67. data/src/core/ext/filters/client_channel/lb_policy.cc +4 -2
  68. data/src/core/ext/filters/client_channel/lb_policy.h +18 -24
  69. data/src/core/ext/filters/client_channel/lb_policy_factory.h +2 -0
  70. data/src/core/ext/filters/client_channel/lb_policy_registry.cc +8 -1
  71. data/src/core/ext/filters/client_channel/lb_policy_registry.h +6 -1
  72. data/src/core/ext/filters/client_channel/local_subchannel_pool.cc +4 -0
  73. data/src/core/ext/filters/client_channel/local_subchannel_pool.h +1 -0
  74. data/src/core/ext/filters/client_channel/proxy_mapper.h +1 -1
  75. data/src/core/ext/filters/client_channel/proxy_mapper_registry.cc +2 -0
  76. data/src/core/ext/filters/client_channel/proxy_mapper_registry.h +5 -0
  77. data/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc +23 -5
  78. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +226 -300
  79. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +8 -1
  80. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +10 -8
  81. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +1 -1
  82. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +26 -9
  83. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +13 -4
  84. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +2 -1
  85. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +2 -0
  86. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h +2 -1
  87. data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +73 -220
  88. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +13 -14
  89. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +5 -2
  90. data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +59 -13
  91. data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +213 -0
  92. data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +113 -0
  93. data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +15 -11
  94. data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +182 -79
  95. data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +5 -11
  96. data/src/core/ext/filters/client_channel/resolver_result_parsing.h +11 -5
  97. data/src/core/ext/filters/client_channel/retry_filter.cc +60 -5
  98. data/src/core/ext/filters/client_channel/retry_service_config.cc +8 -9
  99. data/src/core/ext/filters/client_channel/retry_service_config.h +10 -1
  100. data/src/core/ext/filters/client_channel/retry_throttle.cc +1 -8
  101. data/src/core/ext/filters/client_channel/retry_throttle.h +7 -1
  102. data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +22 -5
  103. data/src/core/ext/filters/client_channel/subchannel.cc +161 -165
  104. data/src/core/ext/filters/client_channel/subchannel.h +80 -20
  105. data/src/core/ext/filters/client_channel/subchannel_interface.h +41 -23
  106. data/src/core/ext/filters/client_channel/subchannel_interface_internal.h +38 -0
  107. data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc +12 -2
  108. data/src/core/ext/filters/client_channel/subchannel_pool_interface.h +6 -3
  109. data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +558 -0
  110. data/src/core/ext/filters/client_channel/subchannel_stream_client.h +226 -0
  111. data/src/core/ext/filters/deadline/deadline_filter.cc +6 -2
  112. data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +123 -366
  113. data/src/core/ext/filters/fault_injection/fault_injection_filter.h +24 -1
  114. data/src/core/ext/filters/http/client/http_client_filter.cc +87 -458
  115. data/src/core/ext/filters/http/client/http_client_filter.h +27 -4
  116. data/src/core/ext/filters/http/client_authority_filter.cc +21 -31
  117. data/src/core/ext/filters/http/client_authority_filter.h +6 -5
  118. data/src/core/ext/filters/http/http_filters_plugin.cc +16 -14
  119. data/src/core/ext/filters/http/message_compress/message_compress_filter.cc +1 -0
  120. data/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +3 -1
  121. data/src/core/ext/filters/http/server/http_server_filter.cc +80 -361
  122. data/src/core/ext/filters/http/server/http_server_filter.h +31 -2
  123. data/src/core/ext/filters/message_size/message_size_filter.cc +20 -26
  124. data/src/core/ext/filters/message_size/message_size_filter.h +2 -2
  125. data/src/core/ext/filters/rbac/rbac_filter.cc +1 -0
  126. data/src/core/ext/filters/server_config_selector/server_config_selector.cc +2 -8
  127. data/src/core/ext/filters/server_config_selector/server_config_selector.h +2 -2
  128. data/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +73 -201
  129. data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +102 -135
  130. data/src/core/ext/transport/chttp2/client/chttp2_connector.h +9 -7
  131. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +49 -19
  132. data/src/core/ext/transport/chttp2/server/chttp2_server.h +2 -0
  133. data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +3 -3
  134. data/src/core/ext/transport/chttp2/transport/bin_decoder.h +2 -1
  135. data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +1 -0
  136. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +205 -45
  137. data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +5 -0
  138. data/src/core/ext/transport/chttp2/transport/context_list.cc +4 -0
  139. data/src/core/ext/transport/chttp2/transport/context_list.h +4 -1
  140. data/src/core/ext/transport/chttp2/transport/flow_control.cc +6 -2
  141. data/src/core/ext/transport/chttp2/transport/flow_control.h +6 -2
  142. data/src/core/ext/transport/chttp2/transport/frame.h +0 -4
  143. data/src/core/ext/transport/chttp2/transport/frame_data.cc +7 -1
  144. data/src/core/ext/transport/chttp2/transport/frame_data.h +4 -1
  145. data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +3 -0
  146. data/src/core/ext/transport/chttp2/transport/frame_goaway.h +3 -1
  147. data/src/core/ext/transport/chttp2/transport/frame_ping.cc +5 -0
  148. data/src/core/ext/transport/chttp2/transport/frame_ping.h +3 -0
  149. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +6 -2
  150. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +3 -0
  151. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +14 -3
  152. data/src/core/ext/transport/chttp2/transport/frame_settings.h +4 -0
  153. data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +4 -1
  154. data/src/core/ext/transport/chttp2/transport/frame_window_update.h +3 -0
  155. data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +13 -15
  156. data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +15 -2
  157. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +3 -0
  158. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +3 -0
  159. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +25 -7
  160. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +8 -1
  161. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +55 -37
  162. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +38 -21
  163. data/src/core/ext/transport/chttp2/transport/http2_settings.h +0 -1
  164. data/src/core/ext/transport/chttp2/transport/internal.h +28 -5
  165. data/src/core/ext/transport/chttp2/transport/parsing.cc +35 -6
  166. data/src/core/ext/transport/chttp2/transport/stream_lists.cc +13 -9
  167. data/src/core/ext/transport/chttp2/transport/stream_map.cc +1 -1
  168. data/src/core/ext/transport/chttp2/transport/stream_map.h +1 -0
  169. data/src/core/ext/transport/chttp2/transport/varint.h +2 -0
  170. data/src/core/ext/transport/chttp2/transport/writing.cc +62 -15
  171. data/src/core/ext/transport/inproc/inproc_transport.cc +8 -5
  172. data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.c +27 -24
  173. data/src/core/ext/upb-generated/envoy/admin/v3/certs.upb.h +143 -63
  174. data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.c +37 -35
  175. data/src/core/ext/upb-generated/envoy/admin/v3/clusters.upb.h +177 -77
  176. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.c +115 -111
  177. data/src/core/ext/upb-generated/envoy/admin/v3/config_dump.upb.h +547 -207
  178. data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.c +5 -5
  179. data/src/core/ext/upb-generated/envoy/admin/v3/init_dump.upb.h +26 -13
  180. data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.c +5 -5
  181. data/src/core/ext/upb-generated/envoy/admin/v3/listeners.upb.h +23 -9
  182. data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.c +7 -7
  183. data/src/core/ext/upb-generated/envoy/admin/v3/memory.upb.h +18 -0
  184. data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.c +4 -4
  185. data/src/core/ext/upb-generated/envoy/admin/v3/metrics.upb.h +13 -4
  186. data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.c +4 -4
  187. data/src/core/ext/upb-generated/envoy/admin/v3/mutex_stats.upb.h +9 -0
  188. data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.c +49 -46
  189. data/src/core/ext/upb-generated/envoy/admin/v3/server_info.upb.h +217 -78
  190. data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.c +3 -3
  191. data/src/core/ext/upb-generated/envoy/admin/v3/tap.upb.h +10 -2
  192. data/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.c +12 -12
  193. data/src/core/ext/upb-generated/envoy/annotations/deprecation.upb.h +40 -8
  194. data/src/core/ext/upb-generated/envoy/annotations/resource.upb.c +5 -5
  195. data/src/core/ext/upb-generated/envoy/annotations/resource.upb.h +20 -2
  196. data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.c +55 -55
  197. data/src/core/ext/upb-generated/envoy/config/accesslog/v3/accesslog.upb.h +289 -110
  198. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.c +133 -125
  199. data/src/core/ext/upb-generated/envoy/config/bootstrap/v3/bootstrap.upb.h +656 -267
  200. data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.c +24 -18
  201. data/src/core/ext/upb-generated/envoy/config/cluster/v3/circuit_breaker.upb.h +96 -23
  202. data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.c +185 -173
  203. data/src/core/ext/upb-generated/envoy/config/cluster/v3/cluster.upb.h +860 -309
  204. data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.c +3 -3
  205. data/src/core/ext/upb-generated/envoy/config/cluster/v3/filter.upb.h +10 -2
  206. data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.c +44 -24
  207. data/src/core/ext/upb-generated/envoy/config/cluster/v3/outlier_detection.upb.h +165 -40
  208. data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.c +72 -63
  209. data/src/core/ext/upb-generated/envoy/config/common/matcher/v3/matcher.upb.h +433 -174
  210. data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.c +33 -31
  211. data/src/core/ext/upb-generated/envoy/config/core/v3/address.upb.h +154 -52
  212. data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.c +5 -4
  213. data/src/core/ext/upb-generated/envoy/config/core/v3/backoff.upb.h +14 -4
  214. data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.c +89 -89
  215. data/src/core/ext/upb-generated/envoy/config/core/v3/base.upb.h +460 -166
  216. data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.c +76 -33
  217. data/src/core/ext/upb-generated/envoy/config/core/v3/config_source.upb.h +379 -69
  218. data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.c +2 -2
  219. data/src/core/ext/upb-generated/envoy/config/core/v3/event_service_config.upb.h +14 -6
  220. data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.c +5 -25
  221. data/src/core/ext/upb-generated/envoy/config/core/v3/extension.upb.h +10 -91
  222. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.c +5 -5
  223. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_method_list.upb.h +26 -13
  224. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.c +70 -68
  225. data/src/core/ext/upb-generated/envoy/config/core/v3/grpc_service.upb.h +369 -131
  226. data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.c +94 -65
  227. data/src/core/ext/upb-generated/envoy/config/core/v3/health_check.upb.h +447 -161
  228. data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.c +4 -4
  229. data/src/core/ext/upb-generated/envoy/config/core/v3/http_uri.upb.h +27 -11
  230. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.c +104 -85
  231. data/src/core/ext/upb-generated/envoy/config/core/v3/protocol.upb.h +378 -113
  232. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.c +2 -2
  233. data/src/core/ext/upb-generated/envoy/config/core/v3/proxy_protocol.upb.h +3 -0
  234. data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.c +6 -6
  235. data/src/core/ext/upb-generated/envoy/config/core/v3/resolver.upb.h +30 -13
  236. data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.c +7 -7
  237. data/src/core/ext/upb-generated/envoy/config/core/v3/socket_option.upb.h +43 -16
  238. data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.c +8 -8
  239. data/src/core/ext/upb-generated/envoy/config/core/v3/substitution_format_string.upb.h +58 -24
  240. data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.c +4 -4
  241. data/src/core/ext/upb-generated/envoy/config/core/v3/udp_socket_config.upb.h +14 -4
  242. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.c +18 -18
  243. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint.upb.h +100 -43
  244. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.c +31 -30
  245. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/endpoint_components.upb.h +171 -71
  246. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.c +32 -32
  247. data/src/core/ext/upb-generated/envoy/config/endpoint/v3/load_report.upb.h +196 -95
  248. data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.c +2 -2
  249. data/src/core/ext/upb-generated/envoy/config/listener/v3/api_listener.upb.h +7 -2
  250. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.c +53 -46
  251. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener.upb.h +284 -129
  252. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.c +56 -50
  253. data/src/core/ext/upb-generated/envoy/config/listener/v3/listener_components.upb.h +337 -146
  254. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.c +13 -11
  255. data/src/core/ext/upb-generated/envoy/config/listener/v3/quic_config.upb.h +49 -14
  256. data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.c +4 -4
  257. data/src/core/ext/upb-generated/envoy/config/listener/v3/udp_listener_config.upb.h +14 -4
  258. data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.c +5 -5
  259. data/src/core/ext/upb-generated/envoy/config/metrics/v3/metrics_service.upb.h +20 -4
  260. data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.c +33 -32
  261. data/src/core/ext/upb-generated/envoy/config/metrics/v3/stats.upb.h +205 -86
  262. data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.c +34 -34
  263. data/src/core/ext/upb-generated/envoy/config/overload/v3/overload.upb.h +175 -74
  264. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.c +56 -52
  265. data/src/core/ext/upb-generated/envoy/config/rbac/v3/rbac.upb.h +370 -146
  266. data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.c +27 -23
  267. data/src/core/ext/upb-generated/envoy/config/route/v3/route.upb.h +173 -74
  268. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.c +362 -341
  269. data/src/core/ext/upb-generated/envoy/config/route/v3/route_components.upb.h +2056 -845
  270. data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.c +11 -11
  271. data/src/core/ext/upb-generated/envoy/config/route/v3/scoped_route.upb.h +49 -16
  272. data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.c +56 -50
  273. data/src/core/ext/upb-generated/envoy/config/tap/v3/common.upb.h +293 -111
  274. data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.c +3 -3
  275. data/src/core/ext/upb-generated/envoy/config/trace/v3/datadog.upb.h +6 -0
  276. data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.c +3 -3
  277. data/src/core/ext/upb-generated/envoy/config/trace/v3/dynamic_ot.upb.h +10 -2
  278. data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.c +5 -5
  279. data/src/core/ext/upb-generated/envoy/config/trace/v3/http_tracer.upb.h +26 -10
  280. data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.c +5 -5
  281. data/src/core/ext/upb-generated/envoy/config/trace/v3/lightstep.upb.h +27 -12
  282. data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.c +17 -16
  283. data/src/core/ext/upb-generated/envoy/config/trace/v3/opencensus.upb.h +83 -36
  284. data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.c +2 -2
  285. data/src/core/ext/upb-generated/envoy/config/trace/v3/service.upb.h +7 -2
  286. data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.c +8 -8
  287. data/src/core/ext/upb-generated/envoy/config/trace/v3/skywalking.upb.h +46 -17
  288. data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.c +9 -9
  289. data/src/core/ext/upb-generated/envoy/config/trace/v3/xray.upb.h +38 -12
  290. data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.c +7 -7
  291. data/src/core/ext/upb-generated/envoy/config/trace/v3/zipkin.upb.h +30 -10
  292. data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.c +2 -2
  293. data/src/core/ext/upb-generated/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +10 -6
  294. data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.c +15 -15
  295. data/src/core/ext/upb-generated/envoy/extensions/filters/common/fault/v3/fault.upb.h +71 -28
  296. data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.c +25 -25
  297. data/src/core/ext/upb-generated/envoy/extensions/filters/http/fault/v3/fault.upb.h +149 -65
  298. data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.c +8 -7
  299. data/src/core/ext/upb-generated/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +30 -12
  300. data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.c +9 -9
  301. data/src/core/ext/upb-generated/envoy/extensions/filters/http/router/v3/router.upb.h +42 -15
  302. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.c +173 -144
  303. data/src/core/ext/upb-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +898 -323
  304. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.c +51 -45
  305. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/common.upb.h +261 -116
  306. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.c +13 -13
  307. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +70 -25
  308. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.c +80 -55
  309. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +410 -124
  310. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.c +5 -5
  311. data/src/core/ext/upb-generated/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +23 -9
  312. data/src/core/ext/upb-generated/envoy/service/discovery/v3/ads.upb.c +1 -1
  313. data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.c +44 -44
  314. data/src/core/ext/upb-generated/envoy/service/discovery/v3/discovery.upb.h +261 -122
  315. data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.c +8 -8
  316. data/src/core/ext/upb-generated/envoy/service/load_stats/v3/lrs.upb.h +47 -21
  317. data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.c +32 -32
  318. data/src/core/ext/upb-generated/envoy/service/status/v3/csds.upb.h +175 -66
  319. data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.c +4 -4
  320. data/src/core/ext/upb-generated/envoy/type/http/v3/cookie.upb.h +17 -6
  321. data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.c +8 -8
  322. data/src/core/ext/upb-generated/envoy/type/http/v3/path_transformation.upb.h +38 -17
  323. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.c +8 -8
  324. data/src/core/ext/upb-generated/envoy/type/matcher/v3/http_inputs.upb.h +12 -0
  325. data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.c +7 -7
  326. data/src/core/ext/upb-generated/envoy/type/matcher/v3/metadata.upb.h +43 -18
  327. data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.c +3 -3
  328. data/src/core/ext/upb-generated/envoy/type/matcher/v3/node.upb.h +20 -9
  329. data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.c +3 -3
  330. data/src/core/ext/upb-generated/envoy/type/matcher/v3/number.upb.h +24 -9
  331. data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.c +2 -2
  332. data/src/core/ext/upb-generated/envoy/type/matcher/v3/path.upb.h +14 -6
  333. data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.c +8 -8
  334. data/src/core/ext/upb-generated/envoy/type/matcher/v3/regex.upb.h +40 -16
  335. data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.c +9 -9
  336. data/src/core/ext/upb-generated/envoy/type/matcher/v3/string.upb.h +70 -25
  337. data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.c +5 -5
  338. data/src/core/ext/upb-generated/envoy/type/matcher/v3/struct.upb.h +37 -18
  339. data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.c +12 -12
  340. data/src/core/ext/upb-generated/envoy/type/matcher/v3/value.upb.h +81 -30
  341. data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.c +16 -16
  342. data/src/core/ext/upb-generated/envoy/type/metadata/v3/metadata.upb.h +76 -30
  343. data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.c +20 -20
  344. data/src/core/ext/upb-generated/envoy/type/tracing/v3/custom_tag.upb.h +90 -30
  345. data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.c +7 -7
  346. data/src/core/ext/upb-generated/envoy/type/v3/hash_policy.upb.h +28 -10
  347. data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.c +2 -2
  348. data/src/core/ext/upb-generated/envoy/type/v3/http_status.upb.h +3 -0
  349. data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.c +5 -5
  350. data/src/core/ext/upb-generated/envoy/type/v3/percent.upb.h +13 -4
  351. data/src/core/ext/upb-generated/envoy/type/v3/range.upb.c +9 -9
  352. data/src/core/ext/upb-generated/envoy/type/v3/range.upb.h +18 -0
  353. data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.c +4 -4
  354. data/src/core/ext/upb-generated/envoy/type/v3/semantic_version.upb.h +9 -0
  355. data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.c +5 -5
  356. data/src/core/ext/upb-generated/envoy/type/v3/token_bucket.upb.h +17 -4
  357. data/src/core/ext/upb-generated/google/api/annotations.upb.c +3 -3
  358. data/src/core/ext/upb-generated/google/api/annotations.upb.h +17 -2
  359. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.c +70 -66
  360. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/checked.upb.h +410 -162
  361. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.c +80 -74
  362. data/src/core/ext/upb-generated/google/api/expr/v1alpha1/syntax.upb.h +440 -158
  363. data/src/core/ext/upb-generated/google/api/http.upb.c +17 -17
  364. data/src/core/ext/upb-generated/google/api/http.upb.h +116 -43
  365. data/src/core/ext/upb-generated/google/api/httpbody.upb.c +4 -4
  366. data/src/core/ext/upb-generated/google/api/httpbody.upb.h +19 -7
  367. data/src/core/ext/upb-generated/google/protobuf/any.upb.c +3 -3
  368. data/src/core/ext/upb-generated/google/protobuf/any.upb.h +6 -0
  369. data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c +163 -162
  370. data/src/core/ext/upb-generated/google/protobuf/descriptor.upb.h +1157 -435
  371. data/src/core/ext/upb-generated/google/protobuf/duration.upb.c +3 -3
  372. data/src/core/ext/upb-generated/google/protobuf/duration.upb.h +10 -4
  373. data/src/core/ext/upb-generated/google/protobuf/empty.upb.c +1 -1
  374. data/src/core/ext/upb-generated/google/protobuf/struct.upb.c +15 -15
  375. data/src/core/ext/upb-generated/google/protobuf/struct.upb.h +108 -40
  376. data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c +3 -3
  377. data/src/core/ext/upb-generated/google/protobuf/timestamp.upb.h +10 -4
  378. data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c +18 -18
  379. data/src/core/ext/upb-generated/google/protobuf/wrappers.upb.h +27 -0
  380. data/src/core/ext/upb-generated/google/rpc/status.upb.c +4 -4
  381. data/src/core/ext/upb-generated/google/rpc/status.upb.h +19 -7
  382. data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.c +15 -15
  383. data/src/core/ext/upb-generated/opencensus/proto/trace/v1/trace_config.upb.h +65 -22
  384. data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c +12 -12
  385. data/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.h +49 -12
  386. data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c +69 -65
  387. data/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.h +364 -149
  388. data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c +8 -7
  389. data/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.h +20 -4
  390. data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c +4 -4
  391. data/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.h +6 -0
  392. data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c +31 -31
  393. data/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.h +150 -58
  394. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.c +11 -11
  395. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls.upb.h +51 -18
  396. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.c +175 -0
  397. data/src/core/ext/upb-generated/src/proto/grpc/lookup/v1/rls_config.upb.h +764 -0
  398. data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.c +22 -22
  399. data/src/core/ext/upb-generated/udpa/annotations/migrate.upb.h +97 -10
  400. data/src/core/ext/upb-generated/udpa/annotations/security.upb.c +6 -6
  401. data/src/core/ext/upb-generated/udpa/annotations/security.upb.h +23 -2
  402. data/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.c +3 -3
  403. data/src/core/ext/upb-generated/udpa/annotations/sensitive.upb.h +10 -2
  404. data/src/core/ext/upb-generated/udpa/annotations/status.upb.c +6 -6
  405. data/src/core/ext/upb-generated/udpa/annotations/status.upb.h +27 -6
  406. data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.c +5 -5
  407. data/src/core/ext/upb-generated/udpa/annotations/versioning.upb.h +20 -2
  408. data/src/core/ext/upb-generated/validate/validate.upb.c +261 -250
  409. data/src/core/ext/upb-generated/validate/validate.upb.h +1836 -663
  410. data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.c +22 -22
  411. data/src/core/ext/upb-generated/xds/annotations/v3/migrate.upb.h +97 -10
  412. data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.c +6 -6
  413. data/src/core/ext/upb-generated/xds/annotations/v3/security.upb.h +23 -2
  414. data/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.c +3 -3
  415. data/src/core/ext/upb-generated/xds/annotations/v3/sensitive.upb.h +10 -2
  416. data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.c +18 -18
  417. data/src/core/ext/upb-generated/xds/annotations/v3/status.upb.h +70 -10
  418. data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.c +5 -5
  419. data/src/core/ext/upb-generated/xds/annotations/v3/versioning.upb.h +20 -2
  420. data/src/core/ext/upb-generated/xds/core/v3/authority.upb.c +2 -2
  421. data/src/core/ext/upb-generated/xds/core/v3/authority.upb.h +3 -0
  422. data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.c +8 -8
  423. data/src/core/ext/upb-generated/xds/core/v3/collection_entry.upb.h +38 -12
  424. data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.c +5 -5
  425. data/src/core/ext/upb-generated/xds/core/v3/context_params.upb.h +27 -10
  426. data/src/core/ext/upb-generated/xds/core/v3/extension.upb.c +3 -3
  427. data/src/core/ext/upb-generated/xds/core/v3/extension.upb.h +10 -2
  428. data/src/core/ext/upb-generated/xds/core/v3/resource.upb.c +5 -5
  429. data/src/core/ext/upb-generated/xds/core/v3/resource.upb.h +21 -8
  430. data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.c +10 -10
  431. data/src/core/ext/upb-generated/xds/core/v3/resource_locator.upb.h +71 -30
  432. data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.c +5 -5
  433. data/src/core/ext/upb-generated/xds/core/v3/resource_name.upb.h +16 -2
  434. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.c +12 -12
  435. data/src/core/ext/upb-generated/xds/data/orca/v3/orca_load_report.upb.h +69 -26
  436. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c +47 -0
  437. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.h +109 -0
  438. data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.c +43 -39
  439. data/src/core/ext/upb-generated/xds/type/matcher/v3/matcher.upb.h +255 -103
  440. data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.c +4 -4
  441. data/src/core/ext/upb-generated/xds/type/matcher/v3/regex.upb.h +19 -8
  442. data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.c +9 -9
  443. data/src/core/ext/upb-generated/xds/type/matcher/v3/string.upb.h +70 -25
  444. data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.c +3 -3
  445. data/src/core/ext/upb-generated/xds/type/v3/typed_struct.upb.h +10 -2
  446. data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/circuit_breaker.upbdefs.c +50 -46
  447. data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/cluster.upbdefs.c +195 -185
  448. data/src/core/ext/upbdefs-generated/envoy/config/cluster/v3/outlier_detection.upbdefs.c +16 -12
  449. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.c +129 -94
  450. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/config_source.upbdefs.h +10 -0
  451. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.c +18 -33
  452. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/extension.upbdefs.h +0 -5
  453. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.c +163 -155
  454. data/src/core/ext/upbdefs-generated/envoy/config/core/v3/health_check.upbdefs.h +5 -0
  455. data/src/core/ext/upbdefs-generated/envoy/config/listener/v3/listener_components.upbdefs.c +141 -138
  456. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route.upbdefs.c +26 -20
  457. data/src/core/ext/upbdefs-generated/envoy/config/route/v3/route_components.upbdefs.c +738 -730
  458. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +271 -251
  459. data/src/core/ext/upbdefs-generated/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h +5 -0
  460. data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +207 -193
  461. data/src/core/ext/upbdefs-generated/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h +5 -0
  462. data/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.c +99 -0
  463. data/src/core/ext/upbdefs-generated/src/proto/grpc/lookup/v1/rls_config.upbdefs.h +75 -0
  464. data/src/core/ext/xds/certificate_provider_store.cc +10 -0
  465. data/src/core/ext/xds/certificate_provider_store.h +9 -0
  466. data/src/core/ext/xds/xds_api.cc +20 -0
  467. data/src/core/ext/xds/xds_bootstrap.cc +2 -2
  468. data/src/core/ext/xds/xds_bootstrap.h +2 -0
  469. data/src/core/ext/xds/xds_certificate_provider.cc +5 -0
  470. data/src/core/ext/xds/xds_certificate_provider.h +8 -0
  471. data/src/core/ext/xds/xds_channel_stack_modifier.cc +5 -5
  472. data/src/core/ext/xds/xds_client.cc +125 -109
  473. data/src/core/ext/xds/xds_client.h +13 -5
  474. data/src/core/ext/xds/xds_cluster.cc +106 -16
  475. data/src/core/ext/xds/xds_cluster.h +3 -0
  476. data/src/core/ext/xds/xds_cluster_specifier_plugin.cc +142 -0
  477. data/src/core/ext/xds/xds_cluster_specifier_plugin.h +79 -0
  478. data/src/core/ext/xds/xds_common_types.cc +9 -9
  479. data/src/core/ext/xds/xds_common_types.h +3 -3
  480. data/src/core/ext/xds/xds_endpoint.cc +12 -5
  481. data/src/core/ext/xds/xds_http_fault_filter.cc +4 -4
  482. data/src/core/ext/xds/xds_listener.cc +20 -10
  483. data/src/core/ext/xds/xds_route_config.cc +162 -25
  484. data/src/core/ext/xds/xds_route_config.h +13 -10
  485. data/src/core/ext/xds/xds_server_config_fetcher.cc +19 -23
  486. data/src/core/lib/address_utils/sockaddr_utils.cc +56 -23
  487. data/src/core/lib/address_utils/sockaddr_utils.h +7 -4
  488. data/src/core/lib/avl/avl.h +71 -6
  489. data/src/core/lib/channel/call_finalization.h +4 -0
  490. data/src/core/lib/channel/call_tracer.h +12 -3
  491. data/src/core/lib/channel/channel_args.cc +179 -81
  492. data/src/core/lib/channel/channel_args.h +242 -11
  493. data/src/core/lib/channel/channel_args_preconditioning.cc +5 -11
  494. data/src/core/lib/channel/channel_args_preconditioning.h +4 -4
  495. data/src/core/lib/channel/channel_stack.cc +41 -3
  496. data/src/core/lib/channel/channel_stack.h +51 -8
  497. data/src/core/lib/channel/channel_stack_builder.cc +9 -84
  498. data/src/core/lib/channel/channel_stack_builder.h +35 -26
  499. data/src/core/lib/channel/channel_stack_builder_impl.cc +97 -0
  500. data/src/core/lib/channel/channel_stack_builder_impl.h +45 -0
  501. data/src/core/lib/channel/channel_trace.cc +8 -13
  502. data/src/core/lib/channel/channel_trace.h +6 -3
  503. data/src/core/lib/channel/channelz.cc +8 -13
  504. data/src/core/lib/channel/channelz.h +13 -4
  505. data/src/core/lib/channel/channelz_registry.cc +7 -14
  506. data/src/core/lib/channel/channelz_registry.h +10 -9
  507. data/src/core/lib/channel/connected_channel.cc +21 -31
  508. data/src/core/lib/channel/connected_channel.h +2 -0
  509. data/src/core/lib/channel/promise_based_filter.cc +848 -260
  510. data/src/core/lib/channel/promise_based_filter.h +215 -68
  511. data/src/core/lib/channel/status_util.cc +2 -0
  512. data/src/core/lib/channel/status_util.h +0 -3
  513. data/src/core/lib/compression/compression_internal.cc +46 -17
  514. data/src/core/lib/compression/compression_internal.h +1 -1
  515. data/src/core/lib/config/core_configuration.cc +3 -0
  516. data/src/core/lib/config/core_configuration.h +2 -1
  517. data/src/core/lib/debug/stats_data.cc +2 -6
  518. data/src/core/lib/debug/stats_data.h +18 -21
  519. data/src/core/lib/event_engine/channel_args_endpoint_config.cc +4 -2
  520. data/src/core/lib/event_engine/channel_args_endpoint_config.h +3 -2
  521. data/src/core/lib/event_engine/default_event_engine_factory.cc +8 -2
  522. data/src/core/lib/event_engine/event_engine.cc +12 -2
  523. data/src/core/lib/event_engine/event_engine_factory.h +5 -0
  524. data/src/core/lib/event_engine/handle_containers.h +67 -0
  525. data/src/core/lib/event_engine/iomgr_engine.cc +206 -0
  526. data/src/core/lib/event_engine/iomgr_engine.h +118 -0
  527. data/src/core/lib/event_engine/memory_allocator.cc +12 -4
  528. data/src/core/lib/event_engine/resolved_address.cc +4 -2
  529. data/src/core/lib/event_engine/slice.cc +102 -0
  530. data/src/core/lib/event_engine/slice_buffer.cc +50 -0
  531. data/src/core/{ext/filters/max_age/max_age_filter.h → lib/event_engine/trace.cc} +3 -11
  532. data/src/core/lib/{iomgr/endpoint_pair_event_engine.cc → event_engine/trace.h} +12 -14
  533. data/src/core/lib/gpr/log.cc +5 -0
  534. data/src/core/lib/gpr/tls.h +4 -5
  535. data/src/core/lib/gprpp/bitset.h +17 -1
  536. data/src/core/lib/gprpp/chunked_vector.h +4 -0
  537. data/src/core/lib/gprpp/construct_destruct.h +1 -0
  538. data/src/core/lib/gprpp/cpp_impl_of.h +4 -0
  539. data/src/core/lib/gprpp/dual_ref_counted.h +1 -4
  540. data/src/core/lib/gprpp/examine_stack.h +0 -1
  541. data/src/core/lib/gprpp/fork.cc +3 -6
  542. data/src/core/lib/gprpp/global_config.h +2 -4
  543. data/src/core/lib/gprpp/global_config_env.cc +3 -2
  544. data/src/core/lib/gprpp/global_config_env.h +3 -1
  545. data/src/core/lib/gprpp/global_config_generic.h +0 -4
  546. data/src/core/lib/gprpp/host_port.cc +2 -0
  547. data/src/core/lib/gprpp/manual_constructor.h +0 -1
  548. data/src/core/lib/gprpp/match.h +75 -0
  549. data/src/core/lib/gprpp/memory.h +1 -5
  550. data/src/core/lib/gprpp/orphanable.h +1 -4
  551. data/src/core/lib/gprpp/overload.h +59 -0
  552. data/src/core/lib/gprpp/ref_counted.h +3 -3
  553. data/src/core/lib/gprpp/ref_counted_ptr.h +1 -1
  554. data/src/core/lib/gprpp/single_set_ptr.h +87 -0
  555. data/src/core/lib/gprpp/stat.h +0 -2
  556. data/src/core/lib/gprpp/stat_posix.cc +7 -2
  557. data/src/core/lib/gprpp/status_helper.cc +25 -5
  558. data/src/core/lib/gprpp/status_helper.h +4 -0
  559. data/src/core/lib/gprpp/sync.h +3 -1
  560. data/src/core/lib/gprpp/table.h +10 -0
  561. data/src/core/lib/gprpp/thd.h +2 -5
  562. data/src/core/lib/gprpp/thd_posix.cc +4 -2
  563. data/src/core/lib/gprpp/thd_windows.cc +2 -0
  564. data/src/core/lib/gprpp/time.cc +20 -0
  565. data/src/core/lib/gprpp/time.h +7 -2
  566. data/src/core/lib/gprpp/time_util.cc +4 -0
  567. data/src/core/lib/gprpp/time_util.h +1 -1
  568. data/src/core/lib/gprpp/unique_type_name.h +104 -0
  569. data/src/core/lib/http/format_request.cc +30 -2
  570. data/src/core/lib/http/format_request.h +2 -0
  571. data/src/core/lib/http/httpcli.cc +88 -81
  572. data/src/core/lib/http/httpcli.h +39 -7
  573. data/src/core/lib/http/httpcli_security_connector.cc +6 -7
  574. data/src/core/lib/http/parser.cc +80 -9
  575. data/src/core/lib/http/parser.h +14 -1
  576. data/src/core/lib/iomgr/endpoint.cc +4 -4
  577. data/src/core/lib/iomgr/endpoint.h +6 -4
  578. data/src/core/lib/iomgr/endpoint_cfstream.cc +7 -4
  579. data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -1
  580. data/src/core/lib/iomgr/ev_posix.cc +7 -11
  581. data/src/core/lib/iomgr/exec_ctx.h +1 -2
  582. data/src/core/lib/iomgr/fork_posix.cc +1 -1
  583. data/src/core/lib/iomgr/internal_errqueue.cc +38 -47
  584. data/src/core/lib/iomgr/internal_errqueue.h +1 -6
  585. data/src/core/lib/iomgr/iomgr.cc +0 -1
  586. data/src/core/lib/iomgr/{event_engine/pollset.h → iomgr_fwd.h} +9 -9
  587. data/src/core/lib/iomgr/pollset_set.h +1 -2
  588. data/src/core/lib/iomgr/port.h +25 -8
  589. data/src/core/lib/iomgr/resolve_address.cc +8 -0
  590. data/src/core/lib/iomgr/resolve_address.h +21 -14
  591. data/src/core/lib/iomgr/resolve_address_impl.h +2 -3
  592. data/src/core/lib/iomgr/resolve_address_posix.cc +8 -14
  593. data/src/core/lib/iomgr/resolve_address_posix.h +5 -2
  594. data/src/core/lib/iomgr/resolve_address_windows.cc +7 -14
  595. data/src/core/lib/iomgr/resolve_address_windows.h +5 -2
  596. data/src/core/lib/iomgr/sockaddr.h +2 -3
  597. data/src/core/lib/iomgr/sockaddr_posix.h +2 -0
  598. data/src/core/lib/iomgr/sockaddr_windows.h +2 -0
  599. data/src/core/lib/iomgr/socket_utils_common_posix.cc +3 -2
  600. data/src/core/lib/iomgr/tcp_client_cfstream.cc +10 -2
  601. data/src/core/lib/iomgr/tcp_client_posix.cc +14 -7
  602. data/src/core/lib/iomgr/tcp_client_windows.cc +13 -6
  603. data/src/core/lib/iomgr/tcp_posix.cc +77 -37
  604. data/src/core/lib/iomgr/tcp_server_posix.cc +50 -27
  605. data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +30 -27
  606. data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +7 -4
  607. data/src/core/lib/iomgr/tcp_server_windows.cc +13 -5
  608. data/src/core/lib/iomgr/tcp_windows.cc +7 -4
  609. data/src/core/lib/json/json.h +1 -2
  610. data/src/core/lib/json/json_reader.cc +9 -1
  611. data/src/core/lib/json/json_util.cc +7 -0
  612. data/src/core/lib/json/json_util.h +13 -4
  613. data/src/core/lib/json/json_writer.cc +6 -1
  614. data/src/core/lib/promise/activity.cc +1 -1
  615. data/src/core/lib/promise/activity.h +6 -6
  616. data/src/core/lib/promise/arena_promise.h +11 -1
  617. data/src/core/lib/promise/call_push_pull.h +148 -0
  618. data/src/core/lib/promise/context.h +1 -1
  619. data/src/core/lib/promise/detail/basic_seq.h +2 -2
  620. data/src/core/lib/promise/detail/promise_factory.h +0 -1
  621. data/src/core/lib/promise/detail/status.h +2 -1
  622. data/src/core/lib/promise/intra_activity_waiter.h +49 -0
  623. data/src/core/lib/promise/latch.h +103 -0
  624. data/src/core/lib/promise/loop.h +1 -0
  625. data/src/core/lib/promise/race.h +0 -1
  626. data/src/core/lib/promise/seq.h +0 -2
  627. data/src/core/lib/promise/sleep.cc +2 -0
  628. data/src/core/lib/promise/sleep.h +10 -0
  629. data/src/core/lib/promise/try_seq.h +2 -2
  630. data/src/core/lib/resolver/resolver_factory.h +1 -2
  631. data/src/core/lib/resolver/server_address.cc +9 -3
  632. data/src/core/lib/resolver/server_address.h +4 -4
  633. data/src/core/lib/resource_quota/api.cc +19 -31
  634. data/src/core/lib/resource_quota/api.h +5 -2
  635. data/src/core/lib/resource_quota/arena.cc +0 -6
  636. data/src/core/lib/resource_quota/arena.h +1 -2
  637. data/src/core/lib/resource_quota/memory_quota.cc +47 -5
  638. data/src/core/lib/resource_quota/memory_quota.h +24 -5
  639. data/src/core/lib/resource_quota/resource_quota.h +16 -0
  640. data/src/core/lib/resource_quota/thread_quota.cc +2 -0
  641. data/src/core/lib/resource_quota/thread_quota.h +4 -0
  642. data/src/core/lib/security/authorization/authorization_policy_provider.h +7 -0
  643. data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +12 -12
  644. data/src/core/lib/security/authorization/grpc_server_authz_filter.h +5 -6
  645. data/src/core/lib/security/context/security_context.h +8 -1
  646. data/src/core/lib/security/credentials/alts/alts_credentials.cc +12 -5
  647. data/src/core/lib/security/credentials/alts/alts_credentials.h +4 -0
  648. data/src/core/lib/security/credentials/call_creds_util.cc +5 -3
  649. data/src/core/lib/security/credentials/call_creds_util.h +2 -2
  650. data/src/core/lib/security/credentials/channel_creds_registry.h +1 -0
  651. data/src/core/lib/security/credentials/composite/composite_credentials.cc +19 -10
  652. data/src/core/lib/security/credentials/composite/composite_credentials.h +11 -9
  653. data/src/core/lib/security/credentials/credentials.h +29 -39
  654. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +81 -0
  655. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +8 -0
  656. data/src/core/lib/security/credentials/external/external_account_credentials.cc +3 -0
  657. data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +2 -0
  658. data/src/core/lib/security/credentials/fake/fake_credentials.cc +18 -12
  659. data/src/core/lib/security/credentials/fake/fake_credentials.h +8 -5
  660. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +27 -13
  661. data/src/core/lib/security/credentials/google_default/google_default_credentials.h +4 -4
  662. data/src/core/lib/security/credentials/iam/iam_credentials.cc +8 -4
  663. data/src/core/lib/security/credentials/iam/iam_credentials.h +6 -2
  664. data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +28 -29
  665. data/src/core/lib/security/credentials/insecure/insecure_credentials.h +57 -0
  666. data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +8 -3
  667. data/src/core/lib/security/credentials/jwt/jwt_credentials.h +6 -2
  668. data/src/core/lib/security/credentials/local/local_credentials.cc +12 -6
  669. data/src/core/lib/security/credentials/local/local_credentials.h +4 -0
  670. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +22 -10
  671. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +14 -6
  672. data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +11 -6
  673. data/src/core/lib/security/credentials/plugin/plugin_credentials.h +8 -6
  674. data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +12 -4
  675. data/src/core/lib/security/credentials/ssl/ssl_credentials.h +8 -0
  676. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +10 -0
  677. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +47 -0
  678. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +34 -1
  679. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +66 -3
  680. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +66 -95
  681. data/src/core/lib/security/credentials/tls/tls_credentials.cc +19 -6
  682. data/src/core/lib/security/credentials/tls/tls_credentials.h +5 -5
  683. data/src/core/lib/security/credentials/xds/xds_credentials.cc +57 -50
  684. data/src/core/lib/security/credentials/xds/xds_credentials.h +31 -5
  685. data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +1 -1
  686. data/src/core/lib/security/security_connector/fake/fake_security_connector.h +1 -1
  687. data/src/core/lib/security/security_connector/local/local_security_connector.cc +2 -2
  688. data/src/core/lib/security/security_connector/security_connector.cc +1 -1
  689. data/src/core/lib/security/security_connector/security_connector.h +10 -1
  690. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +5 -3
  691. data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +4 -2
  692. data/src/core/lib/security/transport/auth_filters.h +7 -7
  693. data/src/core/lib/security/transport/client_auth_filter.cc +26 -20
  694. data/src/core/lib/security/transport/secure_endpoint.cc +213 -133
  695. data/src/core/lib/security/transport/secure_endpoint.h +1 -1
  696. data/src/core/lib/security/transport/security_handshaker.cc +11 -9
  697. data/src/core/lib/security/transport/security_handshaker.h +1 -1
  698. data/src/core/lib/security/transport/server_auth_filter.cc +3 -2
  699. data/src/core/lib/service_config/service_config.h +4 -8
  700. data/src/core/lib/service_config/service_config_call_data.h +4 -1
  701. data/src/core/lib/service_config/service_config_impl.cc +7 -0
  702. data/src/core/lib/service_config/service_config_impl.h +9 -2
  703. data/src/core/lib/service_config/service_config_parser.cc +8 -0
  704. data/src/core/lib/service_config/service_config_parser.h +7 -0
  705. data/src/core/lib/slice/b64.cc +1 -1
  706. data/src/core/lib/slice/b64.h +2 -0
  707. data/src/core/lib/slice/percent_encoding.cc +4 -1
  708. data/src/core/lib/slice/percent_encoding.h +0 -6
  709. data/src/core/lib/slice/slice.cc +2 -1
  710. data/src/core/lib/slice/slice.h +10 -5
  711. data/src/core/lib/slice/slice_api.cc +1 -1
  712. data/src/core/lib/slice/slice_buffer.cc +50 -23
  713. data/src/core/lib/slice/slice_buffer.h +106 -0
  714. data/src/core/lib/{iomgr/is_epollexclusive_available.h → slice/slice_buffer_api.cc} +11 -12
  715. data/src/core/lib/slice/slice_internal.h +4 -3
  716. data/src/core/lib/slice/slice_refcount.h +2 -3
  717. data/src/core/lib/slice/slice_refcount_base.h +2 -3
  718. data/src/core/lib/slice/slice_split.cc +3 -0
  719. data/src/core/lib/slice/slice_split.h +0 -4
  720. data/src/core/lib/slice/slice_string_helpers.cc +4 -0
  721. data/src/core/lib/slice/slice_string_helpers.h +1 -4
  722. data/src/core/lib/surface/builtins.cc +7 -2
  723. data/src/core/lib/surface/byte_buffer.cc +7 -1
  724. data/src/core/lib/surface/byte_buffer_reader.cc +4 -4
  725. data/src/core/lib/surface/call.cc +1052 -917
  726. data/src/core/lib/surface/call.h +16 -16
  727. data/src/core/lib/surface/call_details.cc +4 -4
  728. data/src/core/lib/surface/call_log_batch.cc +7 -1
  729. data/src/core/lib/surface/call_test_only.h +4 -1
  730. data/src/core/lib/surface/channel.cc +181 -243
  731. data/src/core/lib/surface/channel.h +94 -57
  732. data/src/core/lib/surface/channel_init.cc +2 -3
  733. data/src/core/lib/surface/channel_init.h +3 -5
  734. data/src/core/lib/surface/channel_ping.cc +8 -2
  735. data/src/core/lib/surface/channel_stack_type.cc +0 -2
  736. data/src/core/lib/surface/channel_stack_type.h +0 -2
  737. data/src/core/lib/surface/completion_queue.cc +14 -6
  738. data/src/core/lib/surface/completion_queue.h +5 -1
  739. data/src/core/lib/surface/completion_queue_factory.cc +1 -0
  740. data/src/core/lib/surface/completion_queue_factory.h +1 -3
  741. data/src/core/lib/surface/event_string.cc +1 -7
  742. data/src/core/lib/surface/event_string.h +1 -1
  743. data/src/core/lib/surface/init.cc +17 -45
  744. data/src/core/lib/surface/init.h +0 -8
  745. data/src/core/lib/surface/lame_client.cc +64 -110
  746. data/src/core/lib/surface/lame_client.h +40 -2
  747. data/src/core/lib/surface/metadata_array.cc +2 -0
  748. data/src/core/lib/surface/server.cc +72 -70
  749. data/src/core/lib/surface/server.h +40 -11
  750. data/src/core/lib/surface/validate_metadata.cc +2 -5
  751. data/src/core/lib/surface/validate_metadata.h +3 -0
  752. data/src/core/lib/surface/version.cc +2 -2
  753. data/src/core/lib/transport/bdp_estimator.cc +3 -1
  754. data/src/core/lib/transport/bdp_estimator.h +2 -3
  755. data/src/core/lib/transport/byte_stream.cc +6 -3
  756. data/src/core/lib/transport/byte_stream.h +5 -1
  757. data/src/core/lib/transport/connectivity_state.cc +6 -4
  758. data/src/core/lib/transport/connectivity_state.h +2 -3
  759. data/src/core/lib/transport/error_utils.cc +4 -2
  760. data/src/core/lib/transport/error_utils.h +5 -1
  761. data/src/core/lib/{channel → transport}/handshaker.cc +9 -4
  762. data/src/core/lib/{channel → transport}/handshaker.h +13 -6
  763. data/src/core/lib/{channel → transport}/handshaker_factory.h +9 -10
  764. data/src/core/lib/{channel → transport}/handshaker_registry.cc +5 -1
  765. data/src/core/lib/{channel → transport}/handshaker_registry.h +5 -4
  766. data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.cc +24 -10
  767. data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.h +3 -3
  768. data/src/core/lib/transport/metadata_batch.cc +287 -0
  769. data/src/core/lib/transport/metadata_batch.h +142 -269
  770. data/src/core/lib/transport/parsed_metadata.cc +2 -0
  771. data/src/core/lib/transport/parsed_metadata.h +10 -3
  772. data/src/core/lib/transport/status_conversion.cc +2 -0
  773. data/src/core/lib/transport/status_conversion.h +2 -2
  774. data/src/core/lib/transport/tcp_connect_handshaker.cc +253 -0
  775. data/src/core/lib/transport/tcp_connect_handshaker.h +39 -0
  776. data/src/core/lib/transport/timeout_encoding.cc +2 -6
  777. data/src/core/lib/transport/timeout_encoding.h +5 -1
  778. data/src/core/lib/transport/transport.cc +18 -17
  779. data/src/core/lib/transport/transport.h +48 -7
  780. data/src/core/lib/transport/transport_impl.h +14 -3
  781. data/src/core/lib/transport/transport_op_string.cc +9 -10
  782. data/src/core/lib/uri/uri_parser.cc +11 -3
  783. data/src/core/lib/uri/uri_parser.h +0 -2
  784. data/src/core/plugin_registry/grpc_plugin_registry.cc +13 -4
  785. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +2 -2
  786. data/src/core/tsi/ssl/session_cache/ssl_session.h +2 -0
  787. data/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc +2 -0
  788. data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +2 -0
  789. data/src/core/tsi/ssl_transport_security.cc +75 -38
  790. data/src/core/tsi/ssl_transport_security.h +8 -2
  791. data/src/core/tsi/transport_security_interface.h +2 -0
  792. data/src/ruby/ext/grpc/extconf.rb +2 -2
  793. data/src/ruby/lib/grpc/generic/active_call.rb +7 -1
  794. data/src/ruby/lib/grpc/grpc.rb +1 -1
  795. data/src/ruby/lib/grpc/version.rb +1 -1
  796. data/src/ruby/pb/generate_proto_ruby.sh +1 -0
  797. data/src/ruby/pb/test/client.rb +769 -0
  798. data/src/ruby/pb/test/server.rb +252 -0
  799. data/src/ruby/pb/test/xds_client.rb +415 -0
  800. data/third_party/abseil-cpp/absl/cleanup/cleanup.h +140 -0
  801. data/third_party/abseil-cpp/absl/cleanup/internal/cleanup.h +100 -0
  802. data/third_party/abseil-cpp/absl/container/flat_hash_set.h +504 -0
  803. data/third_party/upb/third_party/utf8_range/utf8_range.h +1 -1
  804. data/third_party/upb/upb/decode.c +32 -16
  805. data/third_party/upb/upb/def.c +118 -55
  806. data/third_party/upb/upb/def.h +12 -3
  807. data/third_party/upb/upb/encode.c +14 -8
  808. data/third_party/upb/upb/json_encode.c +776 -0
  809. data/third_party/upb/upb/json_encode.h +62 -0
  810. data/third_party/upb/upb/msg.c +5 -7
  811. data/third_party/upb/upb/msg.h +1 -2
  812. data/third_party/upb/upb/msg_internal.h +49 -36
  813. data/third_party/upb/upb/port_def.inc +8 -0
  814. data/third_party/upb/upb/port_undef.inc +1 -0
  815. data/third_party/upb/upb/table.c +10 -6
  816. data/third_party/upb/upb/table_internal.h +2 -0
  817. data/third_party/upb/upb/upb.h +41 -11
  818. data/third_party/zlib/crc32.c +966 -292
  819. data/third_party/zlib/crc32.h +9441 -436
  820. data/third_party/zlib/deflate.c +78 -30
  821. data/third_party/zlib/deflate.h +12 -15
  822. data/third_party/zlib/gzguts.h +3 -2
  823. data/third_party/zlib/gzlib.c +5 -3
  824. data/third_party/zlib/gzread.c +5 -7
  825. data/third_party/zlib/gzwrite.c +25 -13
  826. data/third_party/zlib/infback.c +2 -1
  827. data/third_party/zlib/inffast.c +14 -14
  828. data/third_party/zlib/inflate.c +39 -8
  829. data/third_party/zlib/inflate.h +3 -2
  830. data/third_party/zlib/inftrees.c +3 -3
  831. data/third_party/zlib/trees.c +27 -48
  832. data/third_party/zlib/zlib.h +123 -100
  833. data/third_party/zlib/zutil.c +2 -2
  834. data/third_party/zlib/zutil.h +12 -9
  835. metadata +100 -72
  836. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_event_engine.cc +0 -31
  837. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_event_engine.cc +0 -28
  838. data/src/core/ext/filters/client_idle/client_idle_filter.cc +0 -201
  839. data/src/core/ext/filters/max_age/max_age_filter.cc +0 -566
  840. data/src/core/lib/event_engine/sockaddr.cc +0 -40
  841. data/src/core/lib/event_engine/sockaddr.h +0 -44
  842. data/src/core/lib/gprpp/capture.h +0 -76
  843. data/src/core/lib/iomgr/ev_epollex_linux.cc +0 -1657
  844. data/src/core/lib/iomgr/ev_epollex_linux.h +0 -30
  845. data/src/core/lib/iomgr/event_engine/closure.cc +0 -77
  846. data/src/core/lib/iomgr/event_engine/closure.h +0 -42
  847. data/src/core/lib/iomgr/event_engine/endpoint.cc +0 -172
  848. data/src/core/lib/iomgr/event_engine/endpoint.h +0 -52
  849. data/src/core/lib/iomgr/event_engine/iomgr.cc +0 -85
  850. data/src/core/lib/iomgr/event_engine/pollset.cc +0 -87
  851. data/src/core/lib/iomgr/event_engine/promise.h +0 -51
  852. data/src/core/lib/iomgr/event_engine/resolved_address_internal.cc +0 -47
  853. data/src/core/lib/iomgr/event_engine/resolved_address_internal.h +0 -37
  854. data/src/core/lib/iomgr/event_engine/resolver.cc +0 -133
  855. data/src/core/lib/iomgr/event_engine/resolver.h +0 -56
  856. data/src/core/lib/iomgr/event_engine/tcp.cc +0 -296
  857. data/src/core/lib/iomgr/event_engine/timer.cc +0 -62
  858. data/src/core/lib/iomgr/is_epollexclusive_available.cc +0 -119
  859. data/src/core/lib/iomgr/sys_epoll_wrapper.h +0 -30
@@ -16,7 +16,20 @@
16
16
 
17
17
  #include "src/core/lib/channel/promise_based_filter.h"
18
18
 
19
+ #include <memory>
20
+ #include <string>
21
+
22
+ #include "absl/base/attributes.h"
23
+ #include "absl/memory/memory.h"
24
+ #include "absl/types/variant.h"
25
+ #include "absl/utility/utility.h"
26
+
27
+ #include <grpc/status.h>
28
+
19
29
  #include "src/core/lib/channel/channel_stack.h"
30
+ #include "src/core/lib/gprpp/manual_constructor.h"
31
+ #include "src/core/lib/iomgr/error.h"
32
+ #include "src/core/lib/slice/slice.h"
20
33
 
21
34
  namespace grpc_core {
22
35
  namespace promise_filter_detail {
@@ -24,6 +37,25 @@ namespace promise_filter_detail {
24
37
  ///////////////////////////////////////////////////////////////////////////////
25
38
  // BaseCallData
26
39
 
40
+ BaseCallData::BaseCallData(grpc_call_element* elem,
41
+ const grpc_call_element_args* args, uint8_t flags)
42
+ : call_stack_(args->call_stack),
43
+ elem_(elem),
44
+ arena_(args->arena),
45
+ call_combiner_(args->call_combiner),
46
+ deadline_(args->deadline),
47
+ context_(args->context) {
48
+ if (flags & kFilterExaminesServerInitialMetadata) {
49
+ server_initial_metadata_latch_ = arena_->New<Latch<ServerMetadata*>>();
50
+ }
51
+ }
52
+
53
+ BaseCallData::~BaseCallData() {
54
+ if (server_initial_metadata_latch_ != nullptr) {
55
+ server_initial_metadata_latch_->~Latch();
56
+ }
57
+ }
58
+
27
59
  // We don't form ActivityPtr's to this type, and consequently don't need
28
60
  // Orphan().
29
61
  void BaseCallData::Orphan() { abort(); }
@@ -49,32 +81,426 @@ void BaseCallData::Wakeup() {
49
81
 
50
82
  void BaseCallData::Drop() { GRPC_CALL_STACK_UNREF(call_stack_, "waker"); }
51
83
 
84
+ ///////////////////////////////////////////////////////////////////////////////
85
+ // BaseCallData::CapturedBatch
86
+
87
+ namespace {
88
+ uintptr_t* RefCountField(grpc_transport_stream_op_batch* b) {
89
+ return &b->handler_private.closure.error_data.scratch;
90
+ }
91
+ } // namespace
92
+
93
+ BaseCallData::CapturedBatch::CapturedBatch() : batch_(nullptr) {}
94
+
95
+ BaseCallData::CapturedBatch::CapturedBatch(
96
+ grpc_transport_stream_op_batch* batch) {
97
+ *RefCountField(batch) = 1;
98
+ batch_ = batch;
99
+ }
100
+
101
+ BaseCallData::CapturedBatch::~CapturedBatch() {
102
+ if (batch_ == nullptr) return;
103
+ // A ref can be dropped by destruction, but it must not release the batch
104
+ uintptr_t& refcnt = *RefCountField(batch_);
105
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
106
+ --refcnt;
107
+ GPR_ASSERT(refcnt != 0);
108
+ }
109
+
110
+ BaseCallData::CapturedBatch::CapturedBatch(const CapturedBatch& rhs)
111
+ : batch_(rhs.batch_) {
112
+ if (batch_ == nullptr) return;
113
+ uintptr_t& refcnt = *RefCountField(batch_);
114
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
115
+ ++refcnt;
116
+ }
117
+
118
+ BaseCallData::CapturedBatch& BaseCallData::CapturedBatch::operator=(
119
+ const CapturedBatch& b) {
120
+ CapturedBatch temp(b);
121
+ Swap(&temp);
122
+ return *this;
123
+ }
124
+
125
+ BaseCallData::CapturedBatch::CapturedBatch(CapturedBatch&& rhs) noexcept
126
+ : batch_(rhs.batch_) {
127
+ rhs.batch_ = nullptr;
128
+ }
129
+
130
+ BaseCallData::CapturedBatch& BaseCallData::CapturedBatch::operator=(
131
+ CapturedBatch&& b) noexcept {
132
+ Swap(&b);
133
+ return *this;
134
+ }
135
+
136
+ void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) {
137
+ auto* batch = absl::exchange(batch_, nullptr);
138
+ GPR_ASSERT(batch != nullptr);
139
+ uintptr_t& refcnt = *RefCountField(batch);
140
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
141
+ if (--refcnt == 0) {
142
+ releaser->Resume(batch);
143
+ }
144
+ }
145
+
146
+ void BaseCallData::CapturedBatch::CompleteWith(Flusher* releaser) {
147
+ auto* batch = absl::exchange(batch_, nullptr);
148
+ GPR_ASSERT(batch != nullptr);
149
+ uintptr_t& refcnt = *RefCountField(batch);
150
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
151
+ if (--refcnt == 0) {
152
+ releaser->Complete(batch);
153
+ }
154
+ }
155
+
156
+ void BaseCallData::CapturedBatch::CancelWith(grpc_error_handle error,
157
+ Flusher* releaser) {
158
+ auto* batch = absl::exchange(batch_, nullptr);
159
+ GPR_ASSERT(batch != nullptr);
160
+ uintptr_t& refcnt = *RefCountField(batch);
161
+ if (refcnt == 0) {
162
+ // refcnt==0 ==> cancelled
163
+ GRPC_ERROR_UNREF(error);
164
+ return;
165
+ }
166
+ refcnt = 0;
167
+ releaser->Cancel(batch, error);
168
+ }
169
+
170
+ ///////////////////////////////////////////////////////////////////////////////
171
+ // BaseCallData::Flusher
172
+
173
+ BaseCallData::Flusher::Flusher(BaseCallData* call) : call_(call) {
174
+ GRPC_CALL_STACK_REF(call_->call_stack(), "flusher");
175
+ }
176
+
177
+ BaseCallData::Flusher::~Flusher() {
178
+ if (release_.empty()) {
179
+ if (call_closures_.size() == 0) {
180
+ GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "nothing to flush");
181
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
182
+ return;
183
+ }
184
+ call_closures_.RunClosures(call_->call_combiner());
185
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
186
+ return;
187
+ }
188
+ auto call_next_op = [](void* p, grpc_error_handle) {
189
+ auto* batch = static_cast<grpc_transport_stream_op_batch*>(p);
190
+ BaseCallData* call =
191
+ static_cast<BaseCallData*>(batch->handler_private.extra_arg);
192
+ grpc_call_next_op(call->elem(), batch);
193
+ GRPC_CALL_STACK_UNREF(call->call_stack(), "flusher_batch");
194
+ };
195
+ for (size_t i = 1; i < release_.size(); i++) {
196
+ auto* batch = release_[i];
197
+ batch->handler_private.extra_arg = call_;
198
+ GRPC_CLOSURE_INIT(&batch->handler_private.closure, call_next_op, batch,
199
+ nullptr);
200
+ GRPC_CALL_STACK_REF(call_->call_stack(), "flusher_batch");
201
+ call_closures_.Add(&batch->handler_private.closure, GRPC_ERROR_NONE,
202
+ "flusher_batch");
203
+ }
204
+ call_closures_.RunClosuresWithoutYielding(call_->call_combiner());
205
+ grpc_call_next_op(call_->elem(), release_[0]);
206
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
207
+ }
208
+
52
209
  ///////////////////////////////////////////////////////////////////////////////
53
210
  // ClientCallData
54
211
 
212
+ struct ClientCallData::RecvInitialMetadata final {
213
+ enum State {
214
+ // Initial state; no op seen
215
+ kInitial,
216
+ // No op seen, but we have a latch that would like to modify it when we do
217
+ kGotLatch,
218
+ // Responded to trailing metadata prior to getting a recv_initial_metadata
219
+ kRespondedToTrailingMetadataPriorToHook,
220
+ // Hooked, no latch yet
221
+ kHookedWaitingForLatch,
222
+ // Hooked, latch seen
223
+ kHookedAndGotLatch,
224
+ // Got the callback, haven't set latch yet
225
+ kCompleteWaitingForLatch,
226
+ // Got the callback and got the latch
227
+ kCompleteAndGotLatch,
228
+ // Got the callback and set the latch
229
+ kCompleteAndSetLatch,
230
+ // Called the original callback
231
+ kResponded,
232
+ };
233
+
234
+ State state = kInitial;
235
+ grpc_closure* original_on_ready = nullptr;
236
+ grpc_closure on_ready;
237
+ grpc_metadata_batch* metadata = nullptr;
238
+ Latch<ServerMetadata*>* server_initial_metadata_publisher = nullptr;
239
+ };
240
+
241
+ class ClientCallData::PollContext {
242
+ public:
243
+ explicit PollContext(ClientCallData* self, Flusher* flusher)
244
+ : self_(self), flusher_(flusher) {
245
+ GPR_ASSERT(self_->poll_ctx_ == nullptr);
246
+
247
+ self_->poll_ctx_ = this;
248
+ scoped_activity_.Init(self_);
249
+ have_scoped_activity_ = true;
250
+ }
251
+
252
+ PollContext(const PollContext&) = delete;
253
+ PollContext& operator=(const PollContext&) = delete;
254
+
255
+ void Run() {
256
+ GPR_ASSERT(have_scoped_activity_);
257
+ repoll_ = false;
258
+ if (self_->server_initial_metadata_latch() != nullptr) {
259
+ switch (self_->recv_initial_metadata_->state) {
260
+ case RecvInitialMetadata::kInitial:
261
+ case RecvInitialMetadata::kGotLatch:
262
+ case RecvInitialMetadata::kHookedWaitingForLatch:
263
+ case RecvInitialMetadata::kHookedAndGotLatch:
264
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
265
+ case RecvInitialMetadata::kResponded:
266
+ case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
267
+ break;
268
+ case RecvInitialMetadata::kCompleteAndGotLatch:
269
+ self_->recv_initial_metadata_->state =
270
+ RecvInitialMetadata::kCompleteAndSetLatch;
271
+ self_->recv_initial_metadata_->server_initial_metadata_publisher->Set(
272
+ self_->recv_initial_metadata_->metadata);
273
+ ABSL_FALLTHROUGH_INTENDED;
274
+ case RecvInitialMetadata::kCompleteAndSetLatch: {
275
+ Poll<ServerMetadata**> p =
276
+ self_->server_initial_metadata_latch()->Wait()();
277
+ if (ServerMetadata*** ppp = absl::get_if<ServerMetadata**>(&p)) {
278
+ ServerMetadata* md = **ppp;
279
+ if (self_->recv_initial_metadata_->metadata != md) {
280
+ *self_->recv_initial_metadata_->metadata = std::move(*md);
281
+ }
282
+ self_->recv_initial_metadata_->state =
283
+ RecvInitialMetadata::kResponded;
284
+ flusher_->AddClosure(
285
+ absl::exchange(self_->recv_initial_metadata_->original_on_ready,
286
+ nullptr),
287
+ GRPC_ERROR_NONE,
288
+ "wake_inside_combiner:recv_initial_metadata_ready");
289
+ }
290
+ } break;
291
+ }
292
+ }
293
+ if (self_->recv_trailing_state_ == RecvTrailingState::kCancelled ||
294
+ self_->recv_trailing_state_ == RecvTrailingState::kResponded) {
295
+ return;
296
+ }
297
+ switch (self_->send_initial_state_) {
298
+ case SendInitialState::kQueued:
299
+ case SendInitialState::kForwarded: {
300
+ // Poll the promise once since we're waiting for it.
301
+ Poll<ServerMetadataHandle> poll = self_->promise_();
302
+ if (auto* r = absl::get_if<ServerMetadataHandle>(&poll)) {
303
+ auto* md = UnwrapMetadata(std::move(*r));
304
+ bool destroy_md = true;
305
+ if (self_->recv_trailing_state_ == RecvTrailingState::kComplete) {
306
+ if (self_->recv_trailing_metadata_ != md) {
307
+ *self_->recv_trailing_metadata_ = std::move(*md);
308
+ } else {
309
+ destroy_md = false;
310
+ }
311
+ self_->recv_trailing_state_ = RecvTrailingState::kResponded;
312
+ flusher_->AddClosure(
313
+ absl::exchange(self_->original_recv_trailing_metadata_ready_,
314
+ nullptr),
315
+ GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:1");
316
+ if (self_->recv_initial_metadata_ != nullptr) {
317
+ switch (self_->recv_initial_metadata_->state) {
318
+ case RecvInitialMetadata::kInitial:
319
+ case RecvInitialMetadata::kGotLatch:
320
+ self_->recv_initial_metadata_->state = RecvInitialMetadata::
321
+ kRespondedToTrailingMetadataPriorToHook;
322
+ break;
323
+ case RecvInitialMetadata::
324
+ kRespondedToTrailingMetadataPriorToHook:
325
+ abort(); // not reachable
326
+ break;
327
+ case RecvInitialMetadata::kHookedWaitingForLatch:
328
+ case RecvInitialMetadata::kHookedAndGotLatch:
329
+ case RecvInitialMetadata::kResponded:
330
+ case RecvInitialMetadata::kCompleteAndGotLatch:
331
+ case RecvInitialMetadata::kCompleteAndSetLatch:
332
+ break;
333
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
334
+ self_->recv_initial_metadata_->state =
335
+ RecvInitialMetadata::kResponded;
336
+ flusher_->AddClosure(
337
+ absl::exchange(
338
+ self_->recv_initial_metadata_->original_on_ready,
339
+ nullptr),
340
+ GRPC_ERROR_CANCELLED,
341
+ "wake_inside_combiner:recv_initial_metadata_ready");
342
+ }
343
+ }
344
+ } else {
345
+ GPR_ASSERT(*md->get_pointer(GrpcStatusMetadata()) !=
346
+ GRPC_STATUS_OK);
347
+ grpc_error_handle error = grpc_error_set_int(
348
+ GRPC_ERROR_CREATE_FROM_STATIC_STRING(
349
+ "early return from promise based filter"),
350
+ GRPC_ERROR_INT_GRPC_STATUS,
351
+ *md->get_pointer(GrpcStatusMetadata()));
352
+ if (auto* message = md->get_pointer(GrpcMessageMetadata())) {
353
+ error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
354
+ message->as_string_view());
355
+ }
356
+ GRPC_ERROR_UNREF(self_->cancelled_error_);
357
+ self_->cancelled_error_ = GRPC_ERROR_REF(error);
358
+ if (self_->recv_initial_metadata_ != nullptr) {
359
+ switch (self_->recv_initial_metadata_->state) {
360
+ case RecvInitialMetadata::kInitial:
361
+ case RecvInitialMetadata::kGotLatch:
362
+ self_->recv_initial_metadata_->state = RecvInitialMetadata::
363
+ kRespondedToTrailingMetadataPriorToHook;
364
+ break;
365
+ case RecvInitialMetadata::kHookedWaitingForLatch:
366
+ case RecvInitialMetadata::kHookedAndGotLatch:
367
+ case RecvInitialMetadata::kResponded:
368
+ break;
369
+ case RecvInitialMetadata::
370
+ kRespondedToTrailingMetadataPriorToHook:
371
+ abort(); // not reachable
372
+ break;
373
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
374
+ case RecvInitialMetadata::kCompleteAndGotLatch:
375
+ case RecvInitialMetadata::kCompleteAndSetLatch:
376
+ self_->recv_initial_metadata_->state =
377
+ RecvInitialMetadata::kResponded;
378
+ flusher_->AddClosure(
379
+ absl::exchange(
380
+ self_->recv_initial_metadata_->original_on_ready,
381
+ nullptr),
382
+ GRPC_ERROR_REF(error),
383
+ "wake_inside_combiner:recv_initial_metadata_ready");
384
+ }
385
+ }
386
+ if (self_->send_initial_state_ == SendInitialState::kQueued) {
387
+ self_->send_initial_state_ = SendInitialState::kCancelled;
388
+ self_->send_initial_metadata_batch_.CancelWith(error, flusher_);
389
+ } else {
390
+ GPR_ASSERT(
391
+ self_->recv_trailing_state_ == RecvTrailingState::kInitial ||
392
+ self_->recv_trailing_state_ == RecvTrailingState::kForwarded);
393
+ self_->call_combiner()->Cancel(GRPC_ERROR_REF(error));
394
+ CapturedBatch b(grpc_make_transport_stream_op(GRPC_CLOSURE_CREATE(
395
+ [](void* p, grpc_error_handle) {
396
+ GRPC_CALL_COMBINER_STOP(static_cast<CallCombiner*>(p),
397
+ "finish_cancel");
398
+ },
399
+ self_->call_combiner(), nullptr)));
400
+ b->cancel_stream = true;
401
+ b->payload->cancel_stream.cancel_error = error;
402
+ b.ResumeWith(flusher_);
403
+ }
404
+ self_->recv_trailing_state_ = RecvTrailingState::kCancelled;
405
+ }
406
+ if (destroy_md) {
407
+ md->~grpc_metadata_batch();
408
+ }
409
+ scoped_activity_.Destroy();
410
+ have_scoped_activity_ = false;
411
+ self_->promise_ = ArenaPromise<ServerMetadataHandle>();
412
+ }
413
+ } break;
414
+ case SendInitialState::kInitial:
415
+ case SendInitialState::kCancelled:
416
+ // If we get a response without sending anything, we just propagate
417
+ // that up. (note: that situation isn't possible once we finish the
418
+ // promise transition).
419
+ if (self_->recv_trailing_state_ == RecvTrailingState::kComplete) {
420
+ self_->recv_trailing_state_ = RecvTrailingState::kResponded;
421
+ flusher_->AddClosure(
422
+ absl::exchange(self_->original_recv_trailing_metadata_ready_,
423
+ nullptr),
424
+ GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:2");
425
+ }
426
+ break;
427
+ }
428
+ }
429
+
430
+ ~PollContext() {
431
+ self_->poll_ctx_ = nullptr;
432
+ if (have_scoped_activity_) scoped_activity_.Destroy();
433
+ if (repoll_) {
434
+ struct NextPoll : public grpc_closure {
435
+ grpc_call_stack* call_stack;
436
+ ClientCallData* call_data;
437
+ };
438
+ auto run = [](void* p, grpc_error_handle) {
439
+ auto* next_poll = static_cast<NextPoll*>(p);
440
+ {
441
+ Flusher flusher(next_poll->call_data);
442
+ next_poll->call_data->WakeInsideCombiner(&flusher);
443
+ }
444
+ GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll");
445
+ delete next_poll;
446
+ };
447
+ // Unique ptr --> release to suppress clang-tidy warnings about allocating
448
+ // in a destructor.
449
+ auto* p = absl::make_unique<NextPoll>().release();
450
+ p->call_stack = self_->call_stack();
451
+ p->call_data = self_;
452
+ GRPC_CALL_STACK_REF(self_->call_stack(), "re-poll");
453
+ GRPC_CLOSURE_INIT(p, run, p, nullptr);
454
+ flusher_->AddClosure(p, GRPC_ERROR_NONE, "re-poll");
455
+ }
456
+ }
457
+
458
+ void Repoll() { repoll_ = true; }
459
+
460
+ void ForwardSendInitialMetadata() {
461
+ self_->send_initial_metadata_batch_.ResumeWith(flusher_);
462
+ }
463
+
464
+ private:
465
+ ManualConstructor<ScopedActivity> scoped_activity_;
466
+ ClientCallData* self_;
467
+ Flusher* flusher_;
468
+ bool repoll_ = false;
469
+ bool have_scoped_activity_;
470
+ };
471
+
55
472
  ClientCallData::ClientCallData(grpc_call_element* elem,
56
- const grpc_call_element_args* args)
57
- : BaseCallData(elem, args) {
473
+ const grpc_call_element_args* args,
474
+ uint8_t flags)
475
+ : BaseCallData(elem, args, flags) {
58
476
  GRPC_CLOSURE_INIT(&recv_trailing_metadata_ready_,
59
477
  RecvTrailingMetadataReadyCallback, this,
60
478
  grpc_schedule_on_exec_ctx);
479
+ if (server_initial_metadata_latch() != nullptr) {
480
+ recv_initial_metadata_ = arena()->New<RecvInitialMetadata>();
481
+ }
61
482
  }
62
483
 
63
484
  ClientCallData::~ClientCallData() {
64
- GPR_ASSERT(!is_polling_);
485
+ GPR_ASSERT(poll_ctx_ == nullptr);
65
486
  GRPC_ERROR_UNREF(cancelled_error_);
487
+ if (recv_initial_metadata_ != nullptr) {
488
+ recv_initial_metadata_->~RecvInitialMetadata();
489
+ }
66
490
  }
67
491
 
68
492
  // Activity implementation.
69
493
  void ClientCallData::ForceImmediateRepoll() {
70
- GPR_ASSERT(is_polling_);
71
- repoll_ = true;
494
+ GPR_ASSERT(poll_ctx_ != nullptr);
495
+ poll_ctx_->Repoll();
72
496
  }
73
497
 
74
498
  // Handle one grpc_transport_stream_op_batch
75
- void ClientCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
499
+ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) {
76
500
  // Fake out the activity based context.
77
501
  ScopedContext context(this);
502
+ CapturedBatch batch(b);
503
+ Flusher flusher(this);
78
504
 
79
505
  // If this is a cancel stream, cancel anything we have pending and propagate
80
506
  // the cancellation.
@@ -84,50 +510,88 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
84
510
  !batch->recv_initial_metadata && !batch->recv_message &&
85
511
  !batch->recv_trailing_metadata);
86
512
  Cancel(batch->payload->cancel_stream.cancel_error);
87
- grpc_call_next_op(elem(), batch);
513
+ if (is_last()) {
514
+ GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
515
+ batch.CompleteWith(&flusher);
516
+ } else {
517
+ batch.ResumeWith(&flusher);
518
+ }
88
519
  return;
89
520
  }
90
521
 
522
+ if (recv_initial_metadata_ != nullptr && batch->recv_initial_metadata) {
523
+ bool hook = true;
524
+ switch (recv_initial_metadata_->state) {
525
+ case RecvInitialMetadata::kInitial:
526
+ recv_initial_metadata_->state =
527
+ RecvInitialMetadata::kHookedWaitingForLatch;
528
+ break;
529
+ case RecvInitialMetadata::kGotLatch:
530
+ recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch;
531
+ break;
532
+ case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
533
+ hook = false;
534
+ break;
535
+ case RecvInitialMetadata::kHookedWaitingForLatch:
536
+ case RecvInitialMetadata::kHookedAndGotLatch:
537
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
538
+ case RecvInitialMetadata::kCompleteAndGotLatch:
539
+ case RecvInitialMetadata::kCompleteAndSetLatch:
540
+ case RecvInitialMetadata::kResponded:
541
+ abort(); // unreachable
542
+ }
543
+ if (hook) {
544
+ auto cb = [](void* ptr, grpc_error_handle error) {
545
+ ClientCallData* self = static_cast<ClientCallData*>(ptr);
546
+ self->RecvInitialMetadataReady(error);
547
+ };
548
+ recv_initial_metadata_->metadata =
549
+ batch->payload->recv_initial_metadata.recv_initial_metadata;
550
+ recv_initial_metadata_->original_on_ready =
551
+ batch->payload->recv_initial_metadata.recv_initial_metadata_ready;
552
+ GRPC_CLOSURE_INIT(&recv_initial_metadata_->on_ready, cb, this, nullptr);
553
+ batch->payload->recv_initial_metadata.recv_initial_metadata_ready =
554
+ &recv_initial_metadata_->on_ready;
555
+ }
556
+ }
557
+
91
558
  // send_initial_metadata: seeing this triggers the start of the promise part
92
559
  // of this filter.
93
560
  if (batch->send_initial_metadata) {
94
561
  // If we're already cancelled, just terminate the batch.
95
562
  if (send_initial_state_ == SendInitialState::kCancelled ||
96
563
  recv_trailing_state_ == RecvTrailingState::kCancelled) {
97
- grpc_transport_stream_op_batch_finish_with_failure(
98
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
99
- return;
100
- }
101
- // Otherwise, we should not have seen a send_initial_metadata op yet.
102
- GPR_ASSERT(send_initial_state_ == SendInitialState::kInitial);
103
- // Mark ourselves as queued.
104
- send_initial_state_ = SendInitialState::kQueued;
105
- if (batch->recv_trailing_metadata) {
106
- // If there's a recv_trailing_metadata op, we queue that too.
107
- GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
108
- recv_trailing_state_ = RecvTrailingState::kQueued;
564
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
565
+ } else {
566
+ // Otherwise, we should not have seen a send_initial_metadata op yet.
567
+ GPR_ASSERT(send_initial_state_ == SendInitialState::kInitial);
568
+ // Mark ourselves as queued.
569
+ send_initial_state_ = SendInitialState::kQueued;
570
+ if (batch->recv_trailing_metadata) {
571
+ // If there's a recv_trailing_metadata op, we queue that too.
572
+ GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
573
+ recv_trailing_state_ = RecvTrailingState::kQueued;
574
+ }
575
+ // This is the queuing!
576
+ send_initial_metadata_batch_ = batch;
577
+ // And kick start the promise.
578
+ StartPromise(&flusher);
109
579
  }
110
- // This is the queuing!
111
- send_initial_metadata_batch_ = batch;
112
- // And kick start the promise.
113
- StartPromise();
114
- return;
115
- }
116
-
117
- // recv_trailing_metadata *without* send_initial_metadata: hook it so we can
118
- // respond to it, and push it down.
119
- if (batch->recv_trailing_metadata) {
580
+ } else if (batch->recv_trailing_metadata) {
581
+ // recv_trailing_metadata *without* send_initial_metadata: hook it so we
582
+ // can respond to it, and push it down.
120
583
  if (recv_trailing_state_ == RecvTrailingState::kCancelled) {
121
- grpc_transport_stream_op_batch_finish_with_failure(
122
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
123
- return;
584
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
585
+ } else {
586
+ GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
587
+ recv_trailing_state_ = RecvTrailingState::kForwarded;
588
+ HookRecvTrailingMetadata(batch);
124
589
  }
125
- GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
126
- recv_trailing_state_ = RecvTrailingState::kForwarded;
127
- HookRecvTrailingMetadata(batch);
590
+ } else if (cancelled_error_ != GRPC_ERROR_NONE) {
591
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
128
592
  }
129
593
 
130
- grpc_call_next_op(elem(), batch);
594
+ if (batch.is_captured()) batch.ResumeWith(&flusher);
131
595
  }
132
596
 
133
597
  // Handle cancellation.
@@ -136,7 +600,7 @@ void ClientCallData::Cancel(grpc_error_handle error) {
136
600
  GRPC_ERROR_UNREF(cancelled_error_);
137
601
  cancelled_error_ = GRPC_ERROR_REF(error);
138
602
  // Stop running the promise.
139
- promise_ = ArenaPromise<TrailingMetadata>();
603
+ promise_ = ArenaPromise<ServerMetadataHandle>();
140
604
  // If we have an op queued, fail that op.
141
605
  // Record what we've done.
142
606
  if (send_initial_state_ == SendInitialState::kQueued) {
@@ -145,52 +609,108 @@ void ClientCallData::Cancel(grpc_error_handle error) {
145
609
  recv_trailing_state_ = RecvTrailingState::kCancelled;
146
610
  }
147
611
  struct FailBatch : public grpc_closure {
148
- grpc_transport_stream_op_batch* batch;
149
- CallCombiner* call_combiner;
612
+ CapturedBatch batch;
613
+ ClientCallData* call;
150
614
  };
151
615
  auto fail = [](void* p, grpc_error_handle error) {
152
616
  auto* f = static_cast<FailBatch*>(p);
153
- grpc_transport_stream_op_batch_finish_with_failure(
154
- f->batch, GRPC_ERROR_REF(error), f->call_combiner);
617
+ {
618
+ Flusher flusher(f->call);
619
+ f->batch.CancelWith(GRPC_ERROR_REF(error), &flusher);
620
+ GRPC_CALL_STACK_UNREF(f->call->call_stack(), "cancel pending batch");
621
+ }
155
622
  delete f;
156
623
  };
157
624
  auto* b = new FailBatch();
158
625
  GRPC_CLOSURE_INIT(b, fail, b, nullptr);
159
- b->batch = absl::exchange(send_initial_metadata_batch_, nullptr);
160
- b->call_combiner = call_combiner();
626
+ b->batch = std::move(send_initial_metadata_batch_);
627
+ b->call = this;
628
+ GRPC_CALL_STACK_REF(call_stack(), "cancel pending batch");
161
629
  GRPC_CALL_COMBINER_START(call_combiner(), b,
162
630
  GRPC_ERROR_REF(cancelled_error_),
163
631
  "cancel pending batch");
164
632
  } else {
165
633
  send_initial_state_ = SendInitialState::kCancelled;
166
634
  }
635
+ if (recv_initial_metadata_ != nullptr) {
636
+ switch (recv_initial_metadata_->state) {
637
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
638
+ case RecvInitialMetadata::kCompleteAndGotLatch:
639
+ case RecvInitialMetadata::kCompleteAndSetLatch:
640
+ recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
641
+ GRPC_CALL_COMBINER_START(
642
+ call_combiner(),
643
+ absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
644
+ GRPC_ERROR_REF(error), "propagate cancellation");
645
+ break;
646
+ case RecvInitialMetadata::kInitial:
647
+ case RecvInitialMetadata::kGotLatch:
648
+ case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
649
+ case RecvInitialMetadata::kHookedWaitingForLatch:
650
+ case RecvInitialMetadata::kHookedAndGotLatch:
651
+ case RecvInitialMetadata::kResponded:
652
+ break;
653
+ }
654
+ }
167
655
  }
168
656
 
169
657
  // Begin running the promise - which will ultimately take some initial
170
658
  // metadata and return some trailing metadata.
171
- void ClientCallData::StartPromise() {
659
+ void ClientCallData::StartPromise(Flusher* flusher) {
172
660
  GPR_ASSERT(send_initial_state_ == SendInitialState::kQueued);
173
661
  ChannelFilter* filter = static_cast<ChannelFilter*>(elem()->channel_data);
174
662
 
175
663
  // Construct the promise.
176
- {
177
- ScopedActivity activity(this);
178
- promise_ = filter->MakeCallPromise(
179
- WrapMetadata(send_initial_metadata_batch_->payload
180
- ->send_initial_metadata.send_initial_metadata),
181
- [this](ClientInitialMetadata initial_metadata) {
182
- return MakeNextPromise(std::move(initial_metadata));
183
- });
664
+ PollContext ctx(this, flusher);
665
+ promise_ = filter->MakeCallPromise(
666
+ CallArgs{WrapMetadata(send_initial_metadata_batch_->payload
667
+ ->send_initial_metadata.send_initial_metadata),
668
+ server_initial_metadata_latch()},
669
+ [this](CallArgs call_args) {
670
+ return MakeNextPromise(std::move(call_args));
671
+ });
672
+ ctx.Run();
673
+ }
674
+
675
+ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) {
676
+ ScopedContext context(this);
677
+ switch (recv_initial_metadata_->state) {
678
+ case RecvInitialMetadata::kHookedWaitingForLatch:
679
+ recv_initial_metadata_->state =
680
+ RecvInitialMetadata::kCompleteWaitingForLatch;
681
+ break;
682
+ case RecvInitialMetadata::kHookedAndGotLatch:
683
+ recv_initial_metadata_->state = RecvInitialMetadata::kCompleteAndGotLatch;
684
+ break;
685
+ case RecvInitialMetadata::kInitial:
686
+ case RecvInitialMetadata::kGotLatch:
687
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
688
+ case RecvInitialMetadata::kCompleteAndGotLatch:
689
+ case RecvInitialMetadata::kCompleteAndSetLatch:
690
+ case RecvInitialMetadata::kResponded:
691
+ case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
692
+ abort(); // unreachable
184
693
  }
185
- // Poll once.
186
- WakeInsideCombiner();
694
+ Flusher flusher(this);
695
+ if (error != GRPC_ERROR_NONE) {
696
+ recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
697
+ flusher.AddClosure(
698
+ absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
699
+ GRPC_ERROR_REF(error), "propagate cancellation");
700
+ } else if (send_initial_state_ == SendInitialState::kCancelled ||
701
+ recv_trailing_state_ == RecvTrailingState::kResponded) {
702
+ recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
703
+ flusher.AddClosure(
704
+ absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
705
+ GRPC_ERROR_REF(cancelled_error_), "propagate cancellation");
706
+ }
707
+ WakeInsideCombiner(&flusher);
187
708
  }
188
709
 
189
710
  // Interject our callback into the op batch for recv trailing metadata ready.
190
711
  // Stash a pointer to the trailing metadata that will be filled in, so we can
191
712
  // manipulate it later.
192
- void ClientCallData::HookRecvTrailingMetadata(
193
- grpc_transport_stream_op_batch* batch) {
713
+ void ClientCallData::HookRecvTrailingMetadata(CapturedBatch batch) {
194
714
  recv_trailing_metadata_ =
195
715
  batch->payload->recv_trailing_metadata.recv_trailing_metadata;
196
716
  original_recv_trailing_metadata_ready_ =
@@ -203,12 +723,46 @@ void ClientCallData::HookRecvTrailingMetadata(
203
723
  // Effectively:
204
724
  // - put the modified initial metadata into the batch to be sent down.
205
725
  // - return a wrapper around PollTrailingMetadata as the promise.
206
- ArenaPromise<TrailingMetadata> ClientCallData::MakeNextPromise(
207
- ClientInitialMetadata initial_metadata) {
726
+ ArenaPromise<ServerMetadataHandle> ClientCallData::MakeNextPromise(
727
+ CallArgs call_args) {
728
+ GPR_ASSERT(poll_ctx_ != nullptr);
208
729
  GPR_ASSERT(send_initial_state_ == SendInitialState::kQueued);
209
730
  send_initial_metadata_batch_->payload->send_initial_metadata
210
- .send_initial_metadata = UnwrapMetadata(std::move(initial_metadata));
211
- return ArenaPromise<TrailingMetadata>(
731
+ .send_initial_metadata =
732
+ UnwrapMetadata(std::move(call_args.client_initial_metadata));
733
+ if (recv_initial_metadata_ != nullptr) {
734
+ // Call args should contain a latch for receiving initial metadata.
735
+ // It might be the one we passed in - in which case we know this filter
736
+ // only wants to examine the metadata, or it might be a new instance, in
737
+ // which case we know the filter wants to mutate.
738
+ GPR_ASSERT(call_args.server_initial_metadata != nullptr);
739
+ recv_initial_metadata_->server_initial_metadata_publisher =
740
+ call_args.server_initial_metadata;
741
+ switch (recv_initial_metadata_->state) {
742
+ case RecvInitialMetadata::kInitial:
743
+ recv_initial_metadata_->state = RecvInitialMetadata::kGotLatch;
744
+ break;
745
+ case RecvInitialMetadata::kHookedWaitingForLatch:
746
+ recv_initial_metadata_->state = RecvInitialMetadata::kHookedAndGotLatch;
747
+ poll_ctx_->Repoll();
748
+ break;
749
+ case RecvInitialMetadata::kCompleteWaitingForLatch:
750
+ recv_initial_metadata_->state =
751
+ RecvInitialMetadata::kCompleteAndGotLatch;
752
+ poll_ctx_->Repoll();
753
+ break;
754
+ case RecvInitialMetadata::kGotLatch:
755
+ case RecvInitialMetadata::kHookedAndGotLatch:
756
+ case RecvInitialMetadata::kCompleteAndGotLatch:
757
+ case RecvInitialMetadata::kCompleteAndSetLatch:
758
+ case RecvInitialMetadata::kResponded:
759
+ case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
760
+ abort(); // unreachable
761
+ }
762
+ } else {
763
+ GPR_ASSERT(call_args.server_initial_metadata == nullptr);
764
+ }
765
+ return ArenaPromise<ServerMetadataHandle>(
212
766
  [this]() { return PollTrailingMetadata(); });
213
767
  }
214
768
 
@@ -216,17 +770,18 @@ ArenaPromise<TrailingMetadata> ClientCallData::MakeNextPromise(
216
770
  // First poll: send the send_initial_metadata op down the stack.
217
771
  // All polls: await receiving the trailing metadata, then return it to the
218
772
  // application.
219
- Poll<TrailingMetadata> ClientCallData::PollTrailingMetadata() {
773
+ Poll<ServerMetadataHandle> ClientCallData::PollTrailingMetadata() {
774
+ GPR_ASSERT(poll_ctx_ != nullptr);
220
775
  if (send_initial_state_ == SendInitialState::kQueued) {
221
776
  // First poll: pass the send_initial_metadata op down the stack.
222
- GPR_ASSERT(send_initial_metadata_batch_ != nullptr);
777
+ GPR_ASSERT(send_initial_metadata_batch_.is_captured());
223
778
  send_initial_state_ = SendInitialState::kForwarded;
224
779
  if (recv_trailing_state_ == RecvTrailingState::kQueued) {
225
780
  // (and the recv_trailing_metadata op if it's part of the queuing)
226
781
  HookRecvTrailingMetadata(send_initial_metadata_batch_);
227
782
  recv_trailing_state_ = RecvTrailingState::kForwarded;
228
783
  }
229
- forward_send_initial_metadata_ = true;
784
+ poll_ctx_->ForwardSendInitialMetadata();
230
785
  }
231
786
  switch (recv_trailing_state_) {
232
787
  case RecvTrailingState::kInitial:
@@ -261,6 +816,17 @@ void ClientCallData::RecvTrailingMetadataReadyCallback(
261
816
  }
262
817
 
263
818
  void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
819
+ Flusher flusher(this);
820
+ // If we were cancelled prior to receiving this callback, we should simply
821
+ // forward the callback up with the same error.
822
+ if (recv_trailing_state_ == RecvTrailingState::kCancelled) {
823
+ if (grpc_closure* call_closure =
824
+ absl::exchange(original_recv_trailing_metadata_ready_, nullptr)) {
825
+ flusher.AddClosure(call_closure, GRPC_ERROR_REF(error),
826
+ "propagate failure");
827
+ }
828
+ return;
829
+ }
264
830
  // If there was an error, we'll put that into the trailing metadata and
265
831
  // proceed as if there was not.
266
832
  if (error != GRPC_ERROR_NONE) {
@@ -271,10 +837,10 @@ void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
271
837
  recv_trailing_state_ = RecvTrailingState::kComplete;
272
838
  // Repoll the promise.
273
839
  ScopedContext context(this);
274
- WakeInsideCombiner();
840
+ WakeInsideCombiner(&flusher);
275
841
  }
276
842
 
277
- // Given an error, fill in TrailingMetadata to represent that error.
843
+ // Given an error, fill in ServerMetadataHandle to represent that error.
278
844
  void ClientCallData::SetStatusFromError(grpc_metadata_batch* metadata,
279
845
  grpc_error_handle error) {
280
846
  grpc_status_code status_code = GRPC_STATUS_UNKNOWN;
@@ -288,160 +854,114 @@ void ClientCallData::SetStatusFromError(grpc_metadata_batch* metadata,
288
854
  }
289
855
 
290
856
  // Wakeup and poll the promise if appropriate.
291
- void ClientCallData::WakeInsideCombiner() {
292
- GPR_ASSERT(!is_polling_);
293
- grpc_closure* call_closure = nullptr;
294
- is_polling_ = true;
295
- grpc_error_handle cancel_send_initial_metadata_error = GRPC_ERROR_NONE;
296
- grpc_transport_stream_op_batch* forward_batch = nullptr;
297
- switch (send_initial_state_) {
298
- case SendInitialState::kQueued:
299
- case SendInitialState::kForwarded: {
300
- // Poll the promise once since we're waiting for it.
301
- Poll<TrailingMetadata> poll;
302
- {
303
- ScopedActivity activity(this);
304
- poll = promise_();
305
- }
306
- if (auto* r = absl::get_if<TrailingMetadata>(&poll)) {
307
- promise_ = ArenaPromise<TrailingMetadata>();
308
- auto* md = UnwrapMetadata(std::move(*r));
309
- bool destroy_md = true;
310
- if (recv_trailing_state_ == RecvTrailingState::kComplete) {
311
- if (recv_trailing_metadata_ != md) {
312
- *recv_trailing_metadata_ = std::move(*md);
313
- } else {
314
- destroy_md = false;
315
- }
316
- recv_trailing_state_ = RecvTrailingState::kResponded;
317
- call_closure =
318
- absl::exchange(original_recv_trailing_metadata_ready_, nullptr);
319
- } else {
320
- GPR_ASSERT(*md->get_pointer(GrpcStatusMetadata()) != GRPC_STATUS_OK);
321
- grpc_error_handle error =
322
- grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
323
- "early return from promise based filter"),
324
- GRPC_ERROR_INT_GRPC_STATUS,
325
- *md->get_pointer(GrpcStatusMetadata()));
326
- if (auto* message = md->get_pointer(GrpcMessageMetadata())) {
327
- error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
328
- message->as_string_view());
329
- }
330
- GRPC_ERROR_UNREF(cancelled_error_);
331
- cancelled_error_ = GRPC_ERROR_REF(error);
332
- if (send_initial_state_ == SendInitialState::kQueued) {
333
- send_initial_state_ = SendInitialState::kCancelled;
334
- cancel_send_initial_metadata_error = error;
335
- } else {
336
- GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial ||
337
- recv_trailing_state_ == RecvTrailingState::kForwarded);
338
- call_combiner()->Cancel(GRPC_ERROR_REF(error));
339
- forward_batch = grpc_make_transport_stream_op(GRPC_CLOSURE_CREATE(
340
- [](void*, grpc_error_handle) {}, nullptr, nullptr));
341
- forward_batch->cancel_stream = true;
342
- forward_batch->payload->cancel_stream.cancel_error = error;
343
- }
344
- recv_trailing_state_ = RecvTrailingState::kCancelled;
345
- }
346
- if (destroy_md) {
347
- md->~grpc_metadata_batch();
348
- }
349
- }
350
- } break;
351
- case SendInitialState::kInitial:
352
- case SendInitialState::kCancelled:
353
- // If we get a response without sending anything, we just propagate
354
- // that up. (note: that situation isn't possible once we finish the
355
- // promise transition).
356
- if (recv_trailing_state_ == RecvTrailingState::kComplete) {
357
- recv_trailing_state_ = RecvTrailingState::kResponded;
358
- call_closure =
359
- absl::exchange(original_recv_trailing_metadata_ready_, nullptr);
360
- }
361
- break;
857
+ void ClientCallData::WakeInsideCombiner(Flusher* flusher) {
858
+ PollContext(this, flusher).Run();
859
+ }
860
+
861
+ void ClientCallData::OnWakeup() {
862
+ Flusher flusher(this);
863
+ ScopedContext context(this);
864
+ WakeInsideCombiner(&flusher);
865
+ }
866
+
867
+ ///////////////////////////////////////////////////////////////////////////////
868
+ // ServerCallData
869
+
870
+ struct ServerCallData::SendInitialMetadata {
871
+ enum State {
872
+ kInitial,
873
+ kGotLatch,
874
+ kQueuedWaitingForLatch,
875
+ kQueuedAndGotLatch,
876
+ kQueuedAndSetLatch,
877
+ kForwarded,
878
+ kCancelled,
879
+ };
880
+ State state = kInitial;
881
+ CapturedBatch batch;
882
+ Latch<ServerMetadata*>* server_initial_metadata_publisher = nullptr;
883
+ };
884
+
885
+ class ServerCallData::PollContext {
886
+ public:
887
+ explicit PollContext(ServerCallData* self, Flusher* flusher)
888
+ : self_(self), flusher_(flusher) {
889
+ GPR_ASSERT(self_->poll_ctx_ == nullptr);
890
+ self_->poll_ctx_ = this;
891
+ scoped_activity_.Init(self_);
892
+ have_scoped_activity_ = true;
362
893
  }
363
- GRPC_CALL_STACK_REF(call_stack(), "finish_poll");
364
- is_polling_ = false;
365
- bool in_combiner = true;
366
- bool repoll = absl::exchange(repoll_, false);
367
- if (forward_batch != nullptr) {
368
- GPR_ASSERT(in_combiner);
369
- in_combiner = false;
370
- forward_send_initial_metadata_ = false;
371
- grpc_call_next_op(elem(), forward_batch);
372
- }
373
- if (cancel_send_initial_metadata_error != GRPC_ERROR_NONE) {
374
- GPR_ASSERT(in_combiner);
375
- forward_send_initial_metadata_ = false;
376
- in_combiner = false;
377
- grpc_transport_stream_op_batch_finish_with_failure(
378
- absl::exchange(send_initial_metadata_batch_, nullptr),
379
- cancel_send_initial_metadata_error, call_combiner());
380
- }
381
- if (absl::exchange(forward_send_initial_metadata_, false)) {
382
- GPR_ASSERT(in_combiner);
383
- in_combiner = false;
384
- grpc_call_next_op(elem(),
385
- absl::exchange(send_initial_metadata_batch_, nullptr));
386
- }
387
- if (call_closure != nullptr) {
388
- GPR_ASSERT(in_combiner);
389
- in_combiner = false;
390
- Closure::Run(DEBUG_LOCATION, call_closure, GRPC_ERROR_NONE);
391
- }
392
- if (repoll) {
393
- if (in_combiner) {
394
- WakeInsideCombiner();
395
- } else {
894
+
895
+ PollContext(const PollContext&) = delete;
896
+ PollContext& operator=(const PollContext&) = delete;
897
+
898
+ ~PollContext() {
899
+ self_->poll_ctx_ = nullptr;
900
+ if (have_scoped_activity_) scoped_activity_.Destroy();
901
+ if (repoll_) {
396
902
  struct NextPoll : public grpc_closure {
397
903
  grpc_call_stack* call_stack;
398
- ClientCallData* call_data;
904
+ ServerCallData* call_data;
399
905
  };
400
906
  auto run = [](void* p, grpc_error_handle) {
401
907
  auto* next_poll = static_cast<NextPoll*>(p);
402
- next_poll->call_data->WakeInsideCombiner();
908
+ {
909
+ Flusher flusher(next_poll->call_data);
910
+ next_poll->call_data->WakeInsideCombiner(&flusher);
911
+ }
403
912
  GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll");
404
913
  delete next_poll;
405
914
  };
406
- auto* p = new NextPoll;
407
- GRPC_CALL_STACK_REF(call_stack(), "re-poll");
915
+ auto* p = absl::make_unique<NextPoll>().release();
916
+ p->call_stack = self_->call_stack();
917
+ p->call_data = self_;
918
+ GRPC_CALL_STACK_REF(self_->call_stack(), "re-poll");
408
919
  GRPC_CLOSURE_INIT(p, run, p, nullptr);
409
- GRPC_CALL_COMBINER_START(call_combiner(), p, GRPC_ERROR_NONE, "re-poll");
920
+ flusher_->AddClosure(p, GRPC_ERROR_NONE, "re-poll");
410
921
  }
411
- } else if (in_combiner) {
412
- GRPC_CALL_COMBINER_STOP(call_combiner(), "poll paused");
413
922
  }
414
- GRPC_CALL_STACK_UNREF(call_stack(), "finish_poll");
415
- }
416
923
 
417
- void ClientCallData::OnWakeup() {
418
- ScopedContext context(this);
419
- WakeInsideCombiner();
420
- }
924
+ void Repoll() { repoll_ = true; }
925
+ void ClearRepoll() { repoll_ = false; }
421
926
 
422
- ///////////////////////////////////////////////////////////////////////////////
423
- // ServerCallData
927
+ private:
928
+ ManualConstructor<ScopedActivity> scoped_activity_;
929
+ ServerCallData* const self_;
930
+ Flusher* const flusher_;
931
+ bool repoll_ = false;
932
+ bool have_scoped_activity_;
933
+ };
424
934
 
425
935
  ServerCallData::ServerCallData(grpc_call_element* elem,
426
- const grpc_call_element_args* args)
427
- : BaseCallData(elem, args) {
936
+ const grpc_call_element_args* args,
937
+ uint8_t flags)
938
+ : BaseCallData(elem, args, flags) {
939
+ if (server_initial_metadata_latch() != nullptr) {
940
+ send_initial_metadata_ = arena()->New<SendInitialMetadata>();
941
+ }
428
942
  GRPC_CLOSURE_INIT(&recv_initial_metadata_ready_,
429
943
  RecvInitialMetadataReadyCallback, this,
430
944
  grpc_schedule_on_exec_ctx);
431
945
  }
432
946
 
433
947
  ServerCallData::~ServerCallData() {
434
- GPR_ASSERT(!is_polling_);
948
+ GPR_ASSERT(poll_ctx_ == nullptr);
435
949
  GRPC_ERROR_UNREF(cancelled_error_);
436
950
  }
437
951
 
438
952
  // Activity implementation.
439
- void ServerCallData::ForceImmediateRepoll() { abort(); } // Not implemented.
953
+ void ServerCallData::ForceImmediateRepoll() {
954
+ GPR_ASSERT(poll_ctx_ != nullptr);
955
+ poll_ctx_->Repoll();
956
+ }
440
957
 
441
958
  // Handle one grpc_transport_stream_op_batch
442
- void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
959
+ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) {
443
960
  // Fake out the activity based context.
444
961
  ScopedContext context(this);
962
+ CapturedBatch batch(b);
963
+ Flusher flusher(this);
964
+ bool wake = false;
445
965
 
446
966
  // If this is a cancel stream, cancel anything we have pending and
447
967
  // propagate the cancellation.
@@ -450,8 +970,14 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
450
970
  !batch->send_trailing_metadata && !batch->send_message &&
451
971
  !batch->recv_initial_metadata && !batch->recv_message &&
452
972
  !batch->recv_trailing_metadata);
453
- Cancel(batch->payload->cancel_stream.cancel_error);
454
- grpc_call_next_op(elem(), batch);
973
+ Cancel(GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error),
974
+ &flusher);
975
+ if (is_last()) {
976
+ GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
977
+ batch.CompleteWith(&flusher);
978
+ } else {
979
+ batch.ResumeWith(&flusher);
980
+ }
455
981
  return;
456
982
  }
457
983
 
@@ -473,81 +999,130 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
473
999
  recv_initial_state_ = RecvInitialState::kForwarded;
474
1000
  }
475
1001
 
1002
+ // send_initial_metadata
1003
+ if (send_initial_metadata_ != nullptr && batch->send_initial_metadata) {
1004
+ switch (send_initial_metadata_->state) {
1005
+ case SendInitialMetadata::kInitial:
1006
+ send_initial_metadata_->state =
1007
+ SendInitialMetadata::kQueuedWaitingForLatch;
1008
+ break;
1009
+ case SendInitialMetadata::kGotLatch:
1010
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch;
1011
+ break;
1012
+ case SendInitialMetadata::kCancelled:
1013
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
1014
+ break;
1015
+ case SendInitialMetadata::kQueuedAndGotLatch:
1016
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1017
+ case SendInitialMetadata::kQueuedAndSetLatch:
1018
+ case SendInitialMetadata::kForwarded:
1019
+ abort(); // not reachable
1020
+ }
1021
+ send_initial_metadata_->batch = batch;
1022
+ wake = true;
1023
+ }
1024
+
476
1025
  // send_trailing_metadata
477
- if (batch->send_trailing_metadata) {
1026
+ if (batch.is_captured() && batch->send_trailing_metadata) {
478
1027
  switch (send_trailing_state_) {
479
1028
  case SendTrailingState::kInitial:
480
1029
  send_trailing_metadata_batch_ = batch;
481
1030
  send_trailing_state_ = SendTrailingState::kQueued;
482
- WakeInsideCombiner([this](grpc_error_handle error) {
483
- GPR_ASSERT(send_trailing_state_ == SendTrailingState::kQueued);
484
- Cancel(error);
485
- });
1031
+ wake = true;
486
1032
  break;
487
1033
  case SendTrailingState::kQueued:
488
1034
  case SendTrailingState::kForwarded:
489
1035
  abort(); // unreachable
490
1036
  break;
491
1037
  case SendTrailingState::kCancelled:
492
- grpc_transport_stream_op_batch_finish_with_failure(
493
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
1038
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
494
1039
  break;
495
1040
  }
496
- return;
497
1041
  }
498
1042
 
499
- grpc_call_next_op(elem(), batch);
1043
+ if (wake) WakeInsideCombiner(&flusher);
1044
+ if (batch.is_captured()) batch.ResumeWith(&flusher);
500
1045
  }
501
1046
 
502
1047
  // Handle cancellation.
503
- void ServerCallData::Cancel(grpc_error_handle error) {
1048
+ void ServerCallData::Cancel(grpc_error_handle error, Flusher* flusher) {
504
1049
  // Track the latest reason for cancellation.
505
1050
  GRPC_ERROR_UNREF(cancelled_error_);
506
- cancelled_error_ = GRPC_ERROR_REF(error);
1051
+ cancelled_error_ = error;
507
1052
  // Stop running the promise.
508
- promise_ = ArenaPromise<TrailingMetadata>();
1053
+ promise_ = ArenaPromise<ServerMetadataHandle>();
509
1054
  if (send_trailing_state_ == SendTrailingState::kQueued) {
510
1055
  send_trailing_state_ = SendTrailingState::kCancelled;
511
- struct FailBatch : public grpc_closure {
512
- grpc_transport_stream_op_batch* batch;
513
- CallCombiner* call_combiner;
514
- };
515
- auto fail = [](void* p, grpc_error_handle error) {
516
- auto* f = static_cast<FailBatch*>(p);
517
- grpc_transport_stream_op_batch_finish_with_failure(
518
- f->batch, GRPC_ERROR_REF(error), f->call_combiner);
519
- delete f;
520
- };
521
- auto* b = new FailBatch();
522
- GRPC_CLOSURE_INIT(b, fail, b, nullptr);
523
- b->batch = absl::exchange(send_trailing_metadata_batch_, nullptr);
524
- b->call_combiner = call_combiner();
525
- GRPC_CALL_COMBINER_START(call_combiner(), b,
526
- GRPC_ERROR_REF(cancelled_error_),
527
- "cancel pending batch");
1056
+ send_trailing_metadata_batch_.CancelWith(GRPC_ERROR_REF(error), flusher);
528
1057
  } else {
529
1058
  send_trailing_state_ = SendTrailingState::kCancelled;
530
1059
  }
1060
+ if (send_initial_metadata_ != nullptr) {
1061
+ switch (send_initial_metadata_->state) {
1062
+ case SendInitialMetadata::kInitial:
1063
+ case SendInitialMetadata::kGotLatch:
1064
+ case SendInitialMetadata::kForwarded:
1065
+ case SendInitialMetadata::kCancelled:
1066
+ break;
1067
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1068
+ case SendInitialMetadata::kQueuedAndGotLatch:
1069
+ case SendInitialMetadata::kQueuedAndSetLatch:
1070
+ send_initial_metadata_->batch.CancelWith(GRPC_ERROR_REF(error),
1071
+ flusher);
1072
+ break;
1073
+ }
1074
+ send_initial_metadata_->state = SendInitialMetadata::kCancelled;
1075
+ }
1076
+ if (auto* closure =
1077
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
1078
+ flusher->AddClosure(closure, GRPC_ERROR_REF(error),
1079
+ "original_recv_initial_metadata");
1080
+ }
531
1081
  }
532
1082
 
533
1083
  // Construct a promise that will "call" the next filter.
534
1084
  // Effectively:
535
1085
  // - put the modified initial metadata into the batch being sent up.
536
1086
  // - return a wrapper around PollTrailingMetadata as the promise.
537
- ArenaPromise<TrailingMetadata> ServerCallData::MakeNextPromise(
538
- ClientInitialMetadata initial_metadata) {
1087
+ ArenaPromise<ServerMetadataHandle> ServerCallData::MakeNextPromise(
1088
+ CallArgs call_args) {
539
1089
  GPR_ASSERT(recv_initial_state_ == RecvInitialState::kComplete);
540
- GPR_ASSERT(UnwrapMetadata(std::move(initial_metadata)) ==
1090
+ GPR_ASSERT(UnwrapMetadata(std::move(call_args.client_initial_metadata)) ==
541
1091
  recv_initial_metadata_);
542
1092
  forward_recv_initial_metadata_callback_ = true;
543
- return ArenaPromise<TrailingMetadata>(
1093
+ if (send_initial_metadata_ != nullptr) {
1094
+ GPR_ASSERT(send_initial_metadata_->server_initial_metadata_publisher ==
1095
+ nullptr);
1096
+ GPR_ASSERT(call_args.server_initial_metadata != nullptr);
1097
+ send_initial_metadata_->server_initial_metadata_publisher =
1098
+ call_args.server_initial_metadata;
1099
+ switch (send_initial_metadata_->state) {
1100
+ case SendInitialMetadata::kInitial:
1101
+ send_initial_metadata_->state = SendInitialMetadata::kGotLatch;
1102
+ break;
1103
+ case SendInitialMetadata::kGotLatch:
1104
+ case SendInitialMetadata::kQueuedAndGotLatch:
1105
+ case SendInitialMetadata::kQueuedAndSetLatch:
1106
+ case SendInitialMetadata::kForwarded:
1107
+ abort(); // not reachable
1108
+ break;
1109
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1110
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch;
1111
+ break;
1112
+ case SendInitialMetadata::kCancelled:
1113
+ break;
1114
+ }
1115
+ } else {
1116
+ GPR_ASSERT(call_args.server_initial_metadata == nullptr);
1117
+ }
1118
+ return ArenaPromise<ServerMetadataHandle>(
544
1119
  [this]() { return PollTrailingMetadata(); });
545
1120
  }
546
1121
 
547
1122
  // Wrapper to make it look like we're calling the next filter as a promise.
548
1123
  // All polls: await sending the trailing metadata, then foward it down the
549
1124
  // stack.
550
- Poll<TrailingMetadata> ServerCallData::PollTrailingMetadata() {
1125
+ Poll<ServerMetadataHandle> ServerCallData::PollTrailingMetadata() {
551
1126
  switch (send_trailing_state_) {
552
1127
  case SendTrailingState::kInitial:
553
1128
  return Pending{};
@@ -571,12 +1146,14 @@ void ServerCallData::RecvInitialMetadataReadyCallback(void* arg,
571
1146
  }
572
1147
 
573
1148
  void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
1149
+ Flusher flusher(this);
574
1150
  GPR_ASSERT(recv_initial_state_ == RecvInitialState::kForwarded);
575
1151
  // If there was an error we just propagate that through
576
1152
  if (error != GRPC_ERROR_NONE) {
577
1153
  recv_initial_state_ = RecvInitialState::kResponded;
578
- Closure::Run(DEBUG_LOCATION, original_recv_initial_metadata_ready_,
579
- GRPC_ERROR_REF(error));
1154
+ flusher.AddClosure(
1155
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr),
1156
+ GRPC_ERROR_REF(error), "propagate error");
580
1157
  return;
581
1158
  }
582
1159
  // Record that we've got the callback.
@@ -586,36 +1163,53 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
586
1163
  ScopedContext context(this);
587
1164
  // Construct the promise.
588
1165
  ChannelFilter* filter = static_cast<ChannelFilter*>(elem()->channel_data);
589
- promise_ = filter->MakeCallPromise(
590
- WrapMetadata(recv_initial_metadata_),
591
- [this](ClientInitialMetadata initial_metadata) {
592
- return MakeNextPromise(std::move(initial_metadata));
593
- });
1166
+ promise_ =
1167
+ filter->MakeCallPromise(CallArgs{WrapMetadata(recv_initial_metadata_),
1168
+ server_initial_metadata_latch()},
1169
+ [this](CallArgs call_args) {
1170
+ return MakeNextPromise(std::move(call_args));
1171
+ });
594
1172
  // Poll once.
595
- bool own_error = false;
596
- WakeInsideCombiner([&error, &own_error](grpc_error_handle new_error) {
597
- GPR_ASSERT(error == GRPC_ERROR_NONE);
598
- error = GRPC_ERROR_REF(new_error);
599
- own_error = true;
600
- });
601
- Closure::Run(DEBUG_LOCATION, original_recv_initial_metadata_ready_,
602
- GRPC_ERROR_REF(error));
603
- if (own_error) GRPC_ERROR_UNREF(error);
1173
+ WakeInsideCombiner(&flusher);
1174
+ if (auto* closure =
1175
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
1176
+ flusher.AddClosure(closure, GRPC_ERROR_NONE,
1177
+ "original_recv_initial_metadata");
1178
+ }
604
1179
  }
605
1180
 
606
1181
  // Wakeup and poll the promise if appropriate.
607
- void ServerCallData::WakeInsideCombiner(
608
- absl::FunctionRef<void(grpc_error_handle)> cancel) {
609
- GPR_ASSERT(!is_polling_);
610
- bool forward_send_trailing_metadata = false;
611
- is_polling_ = true;
612
- if (recv_initial_state_ == RecvInitialState::kComplete) {
613
- Poll<TrailingMetadata> poll;
614
- {
615
- ScopedActivity activity(this);
616
- poll = promise_();
1182
+ void ServerCallData::WakeInsideCombiner(Flusher* flusher) {
1183
+ PollContext poll_ctx(this, flusher);
1184
+ if (send_initial_metadata_ != nullptr &&
1185
+ send_initial_metadata_->state ==
1186
+ SendInitialMetadata::kQueuedAndGotLatch) {
1187
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndSetLatch;
1188
+ send_initial_metadata_->server_initial_metadata_publisher->Set(
1189
+ send_initial_metadata_->batch->payload->send_initial_metadata
1190
+ .send_initial_metadata);
1191
+ }
1192
+ poll_ctx.ClearRepoll();
1193
+ if (promise_.has_value()) {
1194
+ Poll<ServerMetadataHandle> poll;
1195
+ poll = promise_();
1196
+ if (send_initial_metadata_ != nullptr &&
1197
+ send_initial_metadata_->state ==
1198
+ SendInitialMetadata::kQueuedAndSetLatch) {
1199
+ Poll<ServerMetadata**> p = server_initial_metadata_latch()->Wait()();
1200
+ if (ServerMetadata*** ppp = absl::get_if<ServerMetadata**>(&p)) {
1201
+ ServerMetadata* md = **ppp;
1202
+ if (send_initial_metadata_->batch->payload->send_initial_metadata
1203
+ .send_initial_metadata != md) {
1204
+ *send_initial_metadata_->batch->payload->send_initial_metadata
1205
+ .send_initial_metadata = std::move(*md);
1206
+ }
1207
+ send_initial_metadata_->state = SendInitialMetadata::kForwarded;
1208
+ send_initial_metadata_->batch.ResumeWith(flusher);
1209
+ }
617
1210
  }
618
- if (auto* r = absl::get_if<TrailingMetadata>(&poll)) {
1211
+ if (auto* r = absl::get_if<ServerMetadataHandle>(&poll)) {
1212
+ promise_ = ArenaPromise<ServerMetadataHandle>();
619
1213
  auto* md = UnwrapMetadata(std::move(*r));
620
1214
  bool destroy_md = true;
621
1215
  switch (send_trailing_state_) {
@@ -627,7 +1221,7 @@ void ServerCallData::WakeInsideCombiner(
627
1221
  } else {
628
1222
  destroy_md = false;
629
1223
  }
630
- forward_send_trailing_metadata = true;
1224
+ send_trailing_metadata_batch_.ResumeWith(flusher);
631
1225
  send_trailing_state_ = SendTrailingState::kForwarded;
632
1226
  } break;
633
1227
  case SendTrailingState::kForwarded:
@@ -644,8 +1238,7 @@ void ServerCallData::WakeInsideCombiner(
644
1238
  error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
645
1239
  message->as_string_view());
646
1240
  }
647
- cancel(error);
648
- GRPC_ERROR_UNREF(error);
1241
+ Cancel(error, flusher);
649
1242
  } break;
650
1243
  case SendTrailingState::kCancelled:
651
1244
  // Nothing to do.
@@ -656,11 +1249,6 @@ void ServerCallData::WakeInsideCombiner(
656
1249
  }
657
1250
  }
658
1251
  }
659
- is_polling_ = false;
660
- if (forward_send_trailing_metadata) {
661
- grpc_call_next_op(elem(),
662
- absl::exchange(send_trailing_metadata_batch_, nullptr));
663
- }
664
1252
  }
665
1253
 
666
1254
  void ServerCallData::OnWakeup() { abort(); } // not implemented