grpc 1.46.3-x86_64-linux → 1.47.0-x86_64-linux

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of grpc might be problematic. Click here for more details.

Files changed (487) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +49 -97
  3. data/include/grpc/event_engine/event_engine.h +42 -7
  4. data/include/grpc/event_engine/memory_allocator.h +0 -15
  5. data/include/grpc/event_engine/port.h +1 -1
  6. data/include/grpc/event_engine/slice.h +286 -0
  7. data/include/grpc/event_engine/slice_buffer.h +112 -0
  8. data/include/grpc/grpc_security.h +11 -0
  9. data/include/grpc/impl/codegen/port_platform.h +100 -36
  10. data/include/grpc/impl/codegen/slice.h +1 -1
  11. data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +18 -30
  12. data/src/core/ext/filters/channel_idle/channel_idle_filter.h +4 -1
  13. data/src/core/ext/filters/client_channel/backend_metric.cc +17 -12
  14. data/src/core/ext/filters/client_channel/backend_metric.h +19 -9
  15. data/src/core/ext/filters/client_channel/backup_poller.cc +5 -5
  16. data/src/core/ext/filters/client_channel/backup_poller.h +2 -4
  17. data/src/core/ext/filters/client_channel/channel_connectivity.cc +42 -20
  18. data/src/core/ext/filters/client_channel/client_channel.cc +60 -17
  19. data/src/core/ext/filters/client_channel/client_channel.h +26 -6
  20. data/src/core/ext/filters/client_channel/client_channel_channelz.cc +3 -5
  21. data/src/core/ext/filters/client_channel/client_channel_channelz.h +12 -2
  22. data/src/core/ext/filters/client_channel/client_channel_factory.cc +5 -0
  23. data/src/core/ext/filters/client_channel/client_channel_factory.h +8 -0
  24. data/src/core/ext/filters/client_channel/client_channel_plugin.cc +4 -13
  25. data/src/core/ext/filters/client_channel/config_selector.cc +1 -0
  26. data/src/core/ext/filters/client_channel/config_selector.h +7 -5
  27. data/src/core/ext/filters/client_channel/connector.h +8 -2
  28. data/src/core/ext/filters/client_channel/dynamic_filters.cc +13 -2
  29. data/src/core/ext/filters/client_channel/dynamic_filters.h +11 -1
  30. data/src/core/ext/filters/client_channel/global_subchannel_pool.cc +2 -0
  31. data/src/core/ext/filters/client_channel/global_subchannel_pool.h +3 -0
  32. data/src/core/ext/filters/client_channel/health/health_check_client.cc +14 -1
  33. data/src/core/ext/filters/client_channel/health/health_check_client.h +2 -0
  34. data/src/core/ext/filters/client_channel/http_proxy.cc +10 -2
  35. data/src/core/ext/filters/client_channel/http_proxy.h +3 -0
  36. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc +6 -2
  37. data/src/core/ext/filters/client_channel/lb_policy/address_filtering.h +1 -0
  38. data/src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h +49 -0
  39. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc +12 -0
  40. data/src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h +5 -0
  41. data/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc +9 -3
  42. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +59 -19
  43. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_balancer_addresses.cc +4 -0
  44. data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h +6 -0
  45. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc +10 -1
  46. data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +6 -3
  47. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.cc +408 -0
  48. data/src/core/ext/filters/client_channel/lb_policy/oob_backend_metric.h +57 -0
  49. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.cc +1038 -0
  50. data/src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h +54 -0
  51. data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +60 -52
  52. data/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc +102 -60
  53. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.cc +157 -96
  54. data/src/core/ext/filters/client_channel/lb_policy/ring_hash/ring_hash.h +2 -0
  55. data/src/core/ext/filters/client_channel/lb_policy/rls/rls.cc +45 -10
  56. data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +182 -171
  57. data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +39 -36
  58. data/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc +30 -5
  59. data/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +99 -23
  60. data/src/core/ext/filters/client_channel/lb_policy/xds/xds.h +6 -0
  61. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_impl.cc +30 -4
  62. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_manager.cc +23 -3
  63. data/src/core/ext/filters/client_channel/lb_policy/xds/xds_cluster_resolver.cc +75 -11
  64. data/src/core/ext/filters/client_channel/lb_policy.cc +4 -2
  65. data/src/core/ext/filters/client_channel/lb_policy.h +18 -24
  66. data/src/core/ext/filters/client_channel/lb_policy_factory.h +2 -0
  67. data/src/core/ext/filters/client_channel/lb_policy_registry.cc +8 -1
  68. data/src/core/ext/filters/client_channel/lb_policy_registry.h +6 -1
  69. data/src/core/ext/filters/client_channel/local_subchannel_pool.cc +4 -0
  70. data/src/core/ext/filters/client_channel/local_subchannel_pool.h +1 -0
  71. data/src/core/ext/filters/client_channel/proxy_mapper.h +1 -1
  72. data/src/core/ext/filters/client_channel/proxy_mapper_registry.cc +2 -0
  73. data/src/core/ext/filters/client_channel/proxy_mapper_registry.h +5 -0
  74. data/src/core/ext/filters/client_channel/resolver/binder/binder_resolver.cc +23 -5
  75. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +134 -43
  76. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +8 -1
  77. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +10 -8
  78. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +1 -1
  79. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +26 -9
  80. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +13 -4
  81. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +2 -1
  82. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc +2 -0
  83. data/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h +2 -1
  84. data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +30 -7
  85. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +13 -14
  86. data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +5 -2
  87. data/src/core/ext/filters/client_channel/resolver/google_c2p/google_c2p_resolver.cc +59 -13
  88. data/src/core/ext/filters/client_channel/resolver/polling_resolver.cc +13 -1
  89. data/src/core/ext/filters/client_channel/resolver/polling_resolver.h +8 -1
  90. data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +15 -11
  91. data/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc +89 -33
  92. data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +5 -11
  93. data/src/core/ext/filters/client_channel/resolver_result_parsing.h +11 -5
  94. data/src/core/ext/filters/client_channel/retry_filter.cc +42 -2
  95. data/src/core/ext/filters/client_channel/retry_service_config.cc +8 -9
  96. data/src/core/ext/filters/client_channel/retry_service_config.h +10 -1
  97. data/src/core/ext/filters/client_channel/retry_throttle.cc +1 -8
  98. data/src/core/ext/filters/client_channel/retry_throttle.h +7 -1
  99. data/src/core/ext/filters/client_channel/service_config_channel_arg_filter.cc +22 -5
  100. data/src/core/ext/filters/client_channel/subchannel.cc +156 -160
  101. data/src/core/ext/filters/client_channel/subchannel.h +80 -20
  102. data/src/core/ext/filters/client_channel/subchannel_interface.h +41 -23
  103. data/src/core/ext/filters/client_channel/subchannel_interface_internal.h +38 -0
  104. data/src/core/ext/filters/client_channel/subchannel_pool_interface.cc +12 -2
  105. data/src/core/ext/filters/client_channel/subchannel_pool_interface.h +6 -3
  106. data/src/core/ext/filters/client_channel/subchannel_stream_client.cc +16 -2
  107. data/src/core/ext/filters/client_channel/subchannel_stream_client.h +12 -0
  108. data/src/core/ext/filters/deadline/deadline_filter.cc +6 -2
  109. data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +1 -4
  110. data/src/core/ext/filters/http/client/http_client_filter.cc +14 -5
  111. data/src/core/ext/filters/http/client/http_client_filter.h +7 -1
  112. data/src/core/ext/filters/http/client_authority_filter.cc +5 -10
  113. data/src/core/ext/filters/http/http_filters_plugin.cc +7 -8
  114. data/src/core/ext/filters/http/message_compress/message_compress_filter.cc +1 -0
  115. data/src/core/ext/filters/http/message_compress/message_decompress_filter.cc +3 -1
  116. data/src/core/ext/filters/http/server/http_server_filter.cc +80 -247
  117. data/src/core/ext/filters/http/server/http_server_filter.h +31 -2
  118. data/src/core/ext/filters/message_size/message_size_filter.cc +20 -26
  119. data/src/core/ext/filters/message_size/message_size_filter.h +2 -2
  120. data/src/core/ext/filters/rbac/rbac_filter.cc +1 -0
  121. data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +101 -135
  122. data/src/core/ext/transport/chttp2/client/chttp2_connector.h +9 -7
  123. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +44 -15
  124. data/src/core/ext/transport/chttp2/server/chttp2_server.h +2 -0
  125. data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +3 -3
  126. data/src/core/ext/transport/chttp2/transport/bin_decoder.h +2 -1
  127. data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +1 -0
  128. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +69 -20
  129. data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +5 -0
  130. data/src/core/ext/transport/chttp2/transport/context_list.cc +4 -0
  131. data/src/core/ext/transport/chttp2/transport/context_list.h +4 -1
  132. data/src/core/ext/transport/chttp2/transport/flow_control.cc +6 -2
  133. data/src/core/ext/transport/chttp2/transport/flow_control.h +6 -2
  134. data/src/core/ext/transport/chttp2/transport/frame.h +0 -4
  135. data/src/core/ext/transport/chttp2/transport/frame_data.cc +7 -1
  136. data/src/core/ext/transport/chttp2/transport/frame_data.h +4 -1
  137. data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +3 -0
  138. data/src/core/ext/transport/chttp2/transport/frame_goaway.h +3 -1
  139. data/src/core/ext/transport/chttp2/transport/frame_ping.cc +5 -0
  140. data/src/core/ext/transport/chttp2/transport/frame_ping.h +3 -0
  141. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +6 -2
  142. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +3 -0
  143. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +12 -3
  144. data/src/core/ext/transport/chttp2/transport/frame_settings.h +4 -0
  145. data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +4 -1
  146. data/src/core/ext/transport/chttp2/transport/frame_window_update.h +3 -0
  147. data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +14 -12
  148. data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +15 -2
  149. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +3 -0
  150. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +3 -0
  151. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +25 -7
  152. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +8 -1
  153. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +55 -37
  154. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +38 -21
  155. data/src/core/ext/transport/chttp2/transport/http2_settings.h +0 -1
  156. data/src/core/ext/transport/chttp2/transport/internal.h +24 -3
  157. data/src/core/ext/transport/chttp2/transport/parsing.cc +27 -6
  158. data/src/core/ext/transport/chttp2/transport/stream_lists.cc +13 -9
  159. data/src/core/ext/transport/chttp2/transport/stream_map.cc +1 -1
  160. data/src/core/ext/transport/chttp2/transport/stream_map.h +1 -0
  161. data/src/core/ext/transport/chttp2/transport/varint.h +2 -0
  162. data/src/core/ext/transport/chttp2/transport/writing.cc +38 -2
  163. data/src/core/ext/transport/inproc/inproc_transport.cc +8 -5
  164. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.c +47 -0
  165. data/src/core/ext/upb-generated/xds/service/orca/v3/orca.upb.h +109 -0
  166. data/src/core/ext/xds/certificate_provider_store.cc +4 -2
  167. data/src/core/ext/xds/certificate_provider_store.h +1 -1
  168. data/src/core/ext/xds/xds_certificate_provider.cc +4 -1
  169. data/src/core/ext/xds/xds_certificate_provider.h +1 -1
  170. data/src/core/ext/xds/xds_channel_stack_modifier.cc +5 -5
  171. data/src/core/ext/xds/xds_client.cc +46 -22
  172. data/src/core/ext/xds/xds_cluster.cc +106 -16
  173. data/src/core/ext/xds/xds_cluster.h +3 -0
  174. data/src/core/ext/xds/xds_http_fault_filter.cc +3 -3
  175. data/src/core/ext/xds/xds_listener.cc +19 -9
  176. data/src/core/ext/xds/xds_server_config_fetcher.cc +2 -1
  177. data/src/core/lib/address_utils/sockaddr_utils.cc +56 -23
  178. data/src/core/lib/address_utils/sockaddr_utils.h +7 -4
  179. data/src/core/lib/avl/avl.h +3 -1
  180. data/src/core/lib/channel/call_finalization.h +4 -0
  181. data/src/core/lib/channel/call_tracer.h +8 -2
  182. data/src/core/lib/channel/channel_args.cc +41 -22
  183. data/src/core/lib/channel/channel_args.h +33 -3
  184. data/src/core/lib/channel/channel_args_preconditioning.cc +3 -3
  185. data/src/core/lib/channel/channel_args_preconditioning.h +3 -2
  186. data/src/core/lib/channel/channel_stack.cc +41 -3
  187. data/src/core/lib/channel/channel_stack.h +49 -3
  188. data/src/core/lib/channel/channel_stack_builder.cc +9 -19
  189. data/src/core/lib/channel/channel_stack_builder.h +15 -27
  190. data/src/core/lib/channel/channel_stack_builder_impl.cc +36 -41
  191. data/src/core/lib/channel/channel_stack_builder_impl.h +3 -6
  192. data/src/core/lib/channel/channel_trace.cc +8 -13
  193. data/src/core/lib/channel/channel_trace.h +6 -3
  194. data/src/core/lib/channel/channelz.cc +8 -13
  195. data/src/core/lib/channel/channelz.h +13 -4
  196. data/src/core/lib/channel/channelz_registry.cc +7 -14
  197. data/src/core/lib/channel/channelz_registry.h +10 -9
  198. data/src/core/lib/channel/connected_channel.cc +21 -31
  199. data/src/core/lib/channel/connected_channel.h +1 -0
  200. data/src/core/lib/channel/promise_based_filter.cc +444 -189
  201. data/src/core/lib/channel/promise_based_filter.h +160 -27
  202. data/src/core/lib/channel/status_util.cc +2 -0
  203. data/src/core/lib/channel/status_util.h +0 -3
  204. data/src/core/lib/compression/compression_internal.cc +45 -10
  205. data/src/core/lib/compression/compression_internal.h +1 -1
  206. data/src/core/lib/config/core_configuration.cc +3 -0
  207. data/src/core/lib/config/core_configuration.h +2 -1
  208. data/src/core/lib/event_engine/channel_args_endpoint_config.cc +4 -2
  209. data/src/core/lib/event_engine/channel_args_endpoint_config.h +3 -2
  210. data/src/core/lib/event_engine/default_event_engine_factory.cc +8 -2
  211. data/src/core/lib/event_engine/event_engine.cc +12 -2
  212. data/src/core/lib/event_engine/event_engine_factory.h +5 -0
  213. data/src/core/lib/event_engine/handle_containers.h +67 -0
  214. data/src/core/lib/event_engine/iomgr_engine.cc +206 -0
  215. data/src/core/lib/event_engine/iomgr_engine.h +118 -0
  216. data/src/core/lib/event_engine/memory_allocator.cc +12 -4
  217. data/src/core/lib/event_engine/resolved_address.cc +4 -2
  218. data/src/core/lib/event_engine/slice.cc +102 -0
  219. data/src/core/lib/event_engine/slice_buffer.cc +50 -0
  220. data/src/core/lib/{iomgr/event_engine/pollset.h → event_engine/trace.cc} +3 -10
  221. data/src/core/lib/{iomgr/endpoint_pair_event_engine.cc → event_engine/trace.h} +12 -14
  222. data/src/core/lib/gpr/log.cc +5 -0
  223. data/src/core/lib/gpr/tls.h +3 -5
  224. data/src/core/lib/gprpp/bitset.h +5 -1
  225. data/src/core/lib/gprpp/chunked_vector.h +4 -0
  226. data/src/core/lib/gprpp/construct_destruct.h +1 -0
  227. data/src/core/lib/gprpp/dual_ref_counted.h +1 -4
  228. data/src/core/lib/gprpp/examine_stack.h +0 -1
  229. data/src/core/lib/gprpp/fork.cc +3 -6
  230. data/src/core/lib/gprpp/global_config.h +2 -4
  231. data/src/core/lib/gprpp/global_config_env.cc +3 -2
  232. data/src/core/lib/gprpp/global_config_env.h +3 -1
  233. data/src/core/lib/gprpp/global_config_generic.h +0 -4
  234. data/src/core/lib/gprpp/host_port.cc +2 -0
  235. data/src/core/lib/gprpp/manual_constructor.h +0 -1
  236. data/src/core/lib/gprpp/match.h +2 -0
  237. data/src/core/lib/gprpp/memory.h +1 -5
  238. data/src/core/lib/gprpp/orphanable.h +1 -4
  239. data/src/core/lib/gprpp/ref_counted.h +1 -3
  240. data/src/core/lib/gprpp/ref_counted_ptr.h +1 -1
  241. data/src/core/lib/gprpp/stat.h +0 -2
  242. data/src/core/lib/gprpp/stat_posix.cc +7 -2
  243. data/src/core/lib/gprpp/status_helper.cc +8 -4
  244. data/src/core/lib/gprpp/status_helper.h +4 -0
  245. data/src/core/lib/gprpp/sync.h +3 -1
  246. data/src/core/lib/gprpp/table.h +10 -0
  247. data/src/core/lib/gprpp/thd.h +2 -5
  248. data/src/core/lib/gprpp/thd_posix.cc +4 -2
  249. data/src/core/lib/gprpp/thd_windows.cc +2 -0
  250. data/src/core/lib/gprpp/time.cc +8 -0
  251. data/src/core/lib/gprpp/time.h +6 -1
  252. data/src/core/lib/gprpp/time_util.cc +4 -0
  253. data/src/core/lib/gprpp/time_util.h +1 -1
  254. data/src/core/lib/gprpp/unique_type_name.h +104 -0
  255. data/src/core/lib/http/format_request.cc +29 -0
  256. data/src/core/lib/http/format_request.h +2 -0
  257. data/src/core/lib/http/httpcli.cc +88 -81
  258. data/src/core/lib/http/httpcli.h +39 -7
  259. data/src/core/lib/http/httpcli_security_connector.cc +3 -4
  260. data/src/core/lib/iomgr/endpoint.cc +4 -4
  261. data/src/core/lib/iomgr/endpoint.h +6 -4
  262. data/src/core/lib/iomgr/endpoint_cfstream.cc +7 -4
  263. data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -1
  264. data/src/core/lib/iomgr/ev_posix.cc +1 -4
  265. data/src/core/lib/iomgr/exec_ctx.h +1 -2
  266. data/src/core/lib/iomgr/internal_errqueue.cc +38 -47
  267. data/src/core/lib/iomgr/internal_errqueue.h +1 -6
  268. data/src/core/lib/iomgr/iomgr.cc +0 -1
  269. data/src/core/{ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_event_engine.cc → lib/iomgr/iomgr_fwd.h} +9 -12
  270. data/src/core/lib/iomgr/pollset_set.h +1 -2
  271. data/src/core/lib/iomgr/port.h +25 -6
  272. data/src/core/lib/iomgr/resolve_address.cc +8 -0
  273. data/src/core/lib/iomgr/resolve_address.h +21 -14
  274. data/src/core/lib/iomgr/resolve_address_impl.h +2 -3
  275. data/src/core/lib/iomgr/resolve_address_posix.cc +8 -14
  276. data/src/core/lib/iomgr/resolve_address_posix.h +5 -2
  277. data/src/core/lib/iomgr/resolve_address_windows.cc +7 -14
  278. data/src/core/lib/iomgr/resolve_address_windows.h +5 -2
  279. data/src/core/lib/iomgr/sockaddr.h +2 -3
  280. data/src/core/lib/iomgr/sockaddr_posix.h +2 -0
  281. data/src/core/lib/iomgr/sockaddr_windows.h +2 -0
  282. data/src/core/lib/iomgr/socket_utils_common_posix.cc +3 -2
  283. data/src/core/lib/iomgr/tcp_client_cfstream.cc +10 -2
  284. data/src/core/lib/iomgr/tcp_client_posix.cc +12 -5
  285. data/src/core/lib/iomgr/tcp_client_windows.cc +13 -6
  286. data/src/core/lib/iomgr/tcp_posix.cc +9 -27
  287. data/src/core/lib/iomgr/tcp_server_posix.cc +26 -12
  288. data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +30 -27
  289. data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +7 -4
  290. data/src/core/lib/iomgr/tcp_server_windows.cc +13 -5
  291. data/src/core/lib/iomgr/tcp_windows.cc +7 -4
  292. data/src/core/lib/json/json.h +1 -2
  293. data/src/core/lib/json/json_reader.cc +9 -1
  294. data/src/core/lib/json/json_util.cc +7 -0
  295. data/src/core/lib/json/json_util.h +10 -1
  296. data/src/core/lib/json/json_writer.cc +6 -1
  297. data/src/core/lib/promise/activity.cc +1 -1
  298. data/src/core/lib/promise/activity.h +6 -6
  299. data/src/core/lib/promise/arena_promise.h +11 -1
  300. data/src/core/lib/promise/call_push_pull.h +4 -0
  301. data/src/core/lib/promise/context.h +1 -1
  302. data/src/core/lib/promise/detail/basic_seq.h +2 -2
  303. data/src/core/lib/promise/detail/promise_factory.h +0 -1
  304. data/src/core/lib/promise/latch.h +0 -1
  305. data/src/core/lib/promise/loop.h +1 -0
  306. data/src/core/lib/promise/race.h +0 -1
  307. data/src/core/lib/promise/seq.h +0 -2
  308. data/src/core/lib/promise/sleep.cc +2 -0
  309. data/src/core/lib/promise/sleep.h +10 -0
  310. data/src/core/lib/promise/try_seq.h +2 -2
  311. data/src/core/lib/resolver/resolver_factory.h +1 -2
  312. data/src/core/lib/resolver/server_address.cc +9 -3
  313. data/src/core/lib/resolver/server_address.h +4 -4
  314. data/src/core/lib/resource_quota/api.cc +14 -1
  315. data/src/core/lib/resource_quota/api.h +4 -1
  316. data/src/core/lib/resource_quota/arena.cc +0 -6
  317. data/src/core/lib/resource_quota/arena.h +1 -2
  318. data/src/core/lib/resource_quota/memory_quota.cc +47 -5
  319. data/src/core/lib/resource_quota/memory_quota.h +24 -5
  320. data/src/core/lib/resource_quota/resource_quota.h +8 -0
  321. data/src/core/lib/resource_quota/thread_quota.cc +2 -0
  322. data/src/core/lib/resource_quota/thread_quota.h +4 -0
  323. data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +3 -0
  324. data/src/core/lib/security/credentials/alts/alts_credentials.cc +8 -2
  325. data/src/core/lib/security/credentials/alts/alts_credentials.h +2 -2
  326. data/src/core/lib/security/credentials/call_creds_util.cc +2 -0
  327. data/src/core/lib/security/credentials/channel_creds_registry.h +1 -0
  328. data/src/core/lib/security/credentials/composite/composite_credentials.cc +7 -3
  329. data/src/core/lib/security/credentials/composite/composite_credentials.h +5 -4
  330. data/src/core/lib/security/credentials/credentials.h +17 -10
  331. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +81 -0
  332. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +8 -0
  333. data/src/core/lib/security/credentials/external/external_account_credentials.cc +3 -0
  334. data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +2 -0
  335. data/src/core/lib/security/credentials/fake/fake_credentials.cc +13 -3
  336. data/src/core/lib/security/credentials/fake/fake_credentials.h +2 -2
  337. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +25 -15
  338. data/src/core/lib/security/credentials/google_default/google_default_credentials.h +2 -2
  339. data/src/core/lib/security/credentials/iam/iam_credentials.cc +4 -1
  340. data/src/core/lib/security/credentials/iam/iam_credentials.h +2 -2
  341. data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +8 -2
  342. data/src/core/lib/security/credentials/insecure/insecure_credentials.h +4 -4
  343. data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +3 -2
  344. data/src/core/lib/security/credentials/jwt/jwt_credentials.h +2 -2
  345. data/src/core/lib/security/credentials/local/local_credentials.cc +8 -2
  346. data/src/core/lib/security/credentials/local/local_credentials.h +2 -2
  347. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +10 -6
  348. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +4 -4
  349. data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +4 -1
  350. data/src/core/lib/security/credentials/plugin/plugin_credentials.h +1 -1
  351. data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +8 -2
  352. data/src/core/lib/security/credentials/ssl/ssl_credentials.h +4 -4
  353. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +10 -0
  354. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +5 -7
  355. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +26 -1
  356. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +28 -4
  357. data/src/core/lib/security/credentials/tls/tls_credentials.cc +10 -0
  358. data/src/core/lib/security/credentials/tls/tls_credentials.h +2 -2
  359. data/src/core/lib/security/credentials/xds/xds_credentials.cc +12 -3
  360. data/src/core/lib/security/credentials/xds/xds_credentials.h +5 -5
  361. data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +1 -1
  362. data/src/core/lib/security/security_connector/fake/fake_security_connector.h +1 -1
  363. data/src/core/lib/security/security_connector/security_connector.cc +1 -1
  364. data/src/core/lib/security/security_connector/security_connector.h +1 -1
  365. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +1 -1
  366. data/src/core/lib/security/transport/client_auth_filter.cc +6 -8
  367. data/src/core/lib/security/transport/secure_endpoint.cc +23 -12
  368. data/src/core/lib/security/transport/security_handshaker.cc +5 -5
  369. data/src/core/lib/security/transport/security_handshaker.h +1 -1
  370. data/src/core/lib/security/transport/server_auth_filter.cc +3 -2
  371. data/src/core/lib/service_config/service_config.h +4 -8
  372. data/src/core/lib/service_config/service_config_call_data.h +4 -1
  373. data/src/core/lib/service_config/service_config_impl.cc +7 -0
  374. data/src/core/lib/service_config/service_config_impl.h +9 -2
  375. data/src/core/lib/service_config/service_config_parser.cc +8 -0
  376. data/src/core/lib/service_config/service_config_parser.h +7 -0
  377. data/src/core/lib/slice/b64.cc +1 -1
  378. data/src/core/lib/slice/b64.h +2 -0
  379. data/src/core/lib/slice/percent_encoding.cc +4 -1
  380. data/src/core/lib/slice/percent_encoding.h +0 -6
  381. data/src/core/lib/slice/slice.cc +2 -1
  382. data/src/core/lib/slice/slice.h +10 -5
  383. data/src/core/lib/slice/slice_api.cc +1 -1
  384. data/src/core/lib/slice/slice_buffer.cc +50 -23
  385. data/src/core/lib/slice/slice_buffer.h +106 -0
  386. data/src/core/lib/slice/slice_buffer_api.cc +35 -0
  387. data/src/core/lib/slice/slice_internal.h +4 -3
  388. data/src/core/lib/slice/slice_refcount.h +2 -3
  389. data/src/core/lib/slice/slice_refcount_base.h +2 -3
  390. data/src/core/lib/slice/slice_split.cc +3 -0
  391. data/src/core/lib/slice/slice_split.h +0 -4
  392. data/src/core/lib/slice/slice_string_helpers.cc +4 -0
  393. data/src/core/lib/slice/slice_string_helpers.h +1 -4
  394. data/src/core/lib/surface/builtins.cc +7 -2
  395. data/src/core/lib/surface/byte_buffer.cc +7 -1
  396. data/src/core/lib/surface/byte_buffer_reader.cc +4 -4
  397. data/src/core/lib/surface/call.cc +41 -26
  398. data/src/core/lib/surface/call.h +16 -2
  399. data/src/core/lib/surface/call_details.cc +4 -4
  400. data/src/core/lib/surface/call_log_batch.cc +7 -1
  401. data/src/core/lib/surface/call_test_only.h +4 -1
  402. data/src/core/lib/surface/channel.cc +179 -242
  403. data/src/core/lib/surface/channel.h +94 -57
  404. data/src/core/lib/surface/channel_init.h +2 -0
  405. data/src/core/lib/surface/channel_ping.cc +8 -2
  406. data/src/core/lib/surface/channel_stack_type.cc +0 -2
  407. data/src/core/lib/surface/channel_stack_type.h +0 -2
  408. data/src/core/lib/surface/completion_queue.cc +14 -6
  409. data/src/core/lib/surface/completion_queue.h +5 -1
  410. data/src/core/lib/surface/completion_queue_factory.cc +1 -0
  411. data/src/core/lib/surface/completion_queue_factory.h +1 -3
  412. data/src/core/lib/surface/event_string.cc +1 -7
  413. data/src/core/lib/surface/event_string.h +1 -1
  414. data/src/core/lib/surface/init.cc +17 -45
  415. data/src/core/lib/surface/init.h +0 -8
  416. data/src/core/lib/surface/lame_client.cc +64 -110
  417. data/src/core/lib/surface/lame_client.h +40 -2
  418. data/src/core/lib/surface/metadata_array.cc +2 -0
  419. data/src/core/lib/surface/server.cc +69 -56
  420. data/src/core/lib/surface/server.h +39 -9
  421. data/src/core/lib/surface/validate_metadata.cc +2 -5
  422. data/src/core/lib/surface/validate_metadata.h +3 -0
  423. data/src/core/lib/surface/version.cc +2 -2
  424. data/src/core/lib/transport/bdp_estimator.cc +3 -1
  425. data/src/core/lib/transport/bdp_estimator.h +2 -3
  426. data/src/core/lib/transport/byte_stream.cc +4 -3
  427. data/src/core/lib/transport/byte_stream.h +5 -1
  428. data/src/core/lib/transport/connectivity_state.cc +6 -4
  429. data/src/core/lib/transport/connectivity_state.h +2 -3
  430. data/src/core/lib/transport/error_utils.cc +4 -2
  431. data/src/core/lib/transport/error_utils.h +5 -1
  432. data/src/core/lib/{channel → transport}/handshaker.cc +9 -4
  433. data/src/core/lib/{channel → transport}/handshaker.h +13 -6
  434. data/src/core/lib/{channel → transport}/handshaker_factory.h +9 -10
  435. data/src/core/lib/{channel → transport}/handshaker_registry.cc +5 -1
  436. data/src/core/lib/{channel → transport}/handshaker_registry.h +5 -4
  437. data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.cc +24 -10
  438. data/src/core/{ext/filters/client_channel → lib/transport}/http_connect_handshaker.h +3 -3
  439. data/src/core/lib/transport/metadata_batch.cc +287 -0
  440. data/src/core/lib/transport/metadata_batch.h +133 -264
  441. data/src/core/lib/transport/parsed_metadata.cc +2 -0
  442. data/src/core/lib/transport/parsed_metadata.h +10 -3
  443. data/src/core/lib/transport/status_conversion.cc +2 -0
  444. data/src/core/lib/transport/status_conversion.h +2 -2
  445. data/src/core/lib/transport/tcp_connect_handshaker.cc +253 -0
  446. data/src/core/lib/transport/tcp_connect_handshaker.h +39 -0
  447. data/src/core/lib/transport/timeout_encoding.cc +2 -6
  448. data/src/core/lib/transport/timeout_encoding.h +5 -1
  449. data/src/core/lib/transport/transport.cc +18 -17
  450. data/src/core/lib/transport/transport.h +28 -2
  451. data/src/core/lib/transport/transport_impl.h +10 -0
  452. data/src/core/lib/transport/transport_op_string.cc +9 -10
  453. data/src/core/lib/uri/uri_parser.cc +11 -3
  454. data/src/core/lib/uri/uri_parser.h +0 -2
  455. data/src/core/plugin_registry/grpc_plugin_registry.cc +11 -0
  456. data/src/core/tsi/ssl/session_cache/ssl_session.h +2 -0
  457. data/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc +2 -0
  458. data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +2 -0
  459. data/src/ruby/ext/grpc/extconf.rb +2 -2
  460. data/src/ruby/lib/grpc/2.5/grpc_c.so +0 -0
  461. data/src/ruby/lib/grpc/2.6/grpc_c.so +0 -0
  462. data/src/ruby/lib/grpc/2.7/grpc_c.so +0 -0
  463. data/src/ruby/lib/grpc/3.0/grpc_c.so +0 -0
  464. data/src/ruby/lib/grpc/3.1/grpc_c.so +0 -0
  465. data/src/ruby/lib/grpc/grpc_c.so +0 -0
  466. data/src/ruby/lib/grpc/version.rb +1 -1
  467. data/third_party/abseil-cpp/absl/cleanup/cleanup.h +140 -0
  468. data/third_party/abseil-cpp/absl/cleanup/internal/cleanup.h +100 -0
  469. data/third_party/abseil-cpp/absl/container/flat_hash_set.h +504 -0
  470. metadata +36 -29
  471. data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_event_engine.cc +0 -31
  472. data/src/core/lib/event_engine/sockaddr.cc +0 -40
  473. data/src/core/lib/event_engine/sockaddr.h +0 -44
  474. data/src/core/lib/gprpp/capture.h +0 -76
  475. data/src/core/lib/iomgr/event_engine/closure.cc +0 -77
  476. data/src/core/lib/iomgr/event_engine/closure.h +0 -42
  477. data/src/core/lib/iomgr/event_engine/endpoint.cc +0 -172
  478. data/src/core/lib/iomgr/event_engine/endpoint.h +0 -52
  479. data/src/core/lib/iomgr/event_engine/iomgr.cc +0 -85
  480. data/src/core/lib/iomgr/event_engine/pollset.cc +0 -87
  481. data/src/core/lib/iomgr/event_engine/promise.h +0 -51
  482. data/src/core/lib/iomgr/event_engine/resolved_address_internal.cc +0 -47
  483. data/src/core/lib/iomgr/event_engine/resolved_address_internal.h +0 -37
  484. data/src/core/lib/iomgr/event_engine/resolver.cc +0 -133
  485. data/src/core/lib/iomgr/event_engine/resolver.h +0 -56
  486. data/src/core/lib/iomgr/event_engine/tcp.cc +0 -296
  487. data/src/core/lib/iomgr/event_engine/timer.cc +0 -62
@@ -16,9 +16,20 @@
16
16
 
17
17
  #include "src/core/lib/channel/promise_based_filter.h"
18
18
 
19
- #include <cstdlib>
19
+ #include <memory>
20
+ #include <string>
21
+
22
+ #include "absl/base/attributes.h"
23
+ #include "absl/memory/memory.h"
24
+ #include "absl/types/variant.h"
25
+ #include "absl/utility/utility.h"
26
+
27
+ #include <grpc/status.h>
20
28
 
21
29
  #include "src/core/lib/channel/channel_stack.h"
30
+ #include "src/core/lib/gprpp/manual_constructor.h"
31
+ #include "src/core/lib/iomgr/error.h"
32
+ #include "src/core/lib/slice/slice.h"
22
33
 
23
34
  namespace grpc_core {
24
35
  namespace promise_filter_detail {
@@ -70,6 +81,131 @@ void BaseCallData::Wakeup() {
70
81
 
71
82
  void BaseCallData::Drop() { GRPC_CALL_STACK_UNREF(call_stack_, "waker"); }
72
83
 
84
+ ///////////////////////////////////////////////////////////////////////////////
85
+ // BaseCallData::CapturedBatch
86
+
87
+ namespace {
88
+ uintptr_t* RefCountField(grpc_transport_stream_op_batch* b) {
89
+ return &b->handler_private.closure.error_data.scratch;
90
+ }
91
+ } // namespace
92
+
93
+ BaseCallData::CapturedBatch::CapturedBatch() : batch_(nullptr) {}
94
+
95
+ BaseCallData::CapturedBatch::CapturedBatch(
96
+ grpc_transport_stream_op_batch* batch) {
97
+ *RefCountField(batch) = 1;
98
+ batch_ = batch;
99
+ }
100
+
101
+ BaseCallData::CapturedBatch::~CapturedBatch() {
102
+ if (batch_ == nullptr) return;
103
+ // A ref can be dropped by destruction, but it must not release the batch
104
+ uintptr_t& refcnt = *RefCountField(batch_);
105
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
106
+ --refcnt;
107
+ GPR_ASSERT(refcnt != 0);
108
+ }
109
+
110
+ BaseCallData::CapturedBatch::CapturedBatch(const CapturedBatch& rhs)
111
+ : batch_(rhs.batch_) {
112
+ if (batch_ == nullptr) return;
113
+ uintptr_t& refcnt = *RefCountField(batch_);
114
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
115
+ ++refcnt;
116
+ }
117
+
118
+ BaseCallData::CapturedBatch& BaseCallData::CapturedBatch::operator=(
119
+ const CapturedBatch& b) {
120
+ CapturedBatch temp(b);
121
+ Swap(&temp);
122
+ return *this;
123
+ }
124
+
125
+ BaseCallData::CapturedBatch::CapturedBatch(CapturedBatch&& rhs) noexcept
126
+ : batch_(rhs.batch_) {
127
+ rhs.batch_ = nullptr;
128
+ }
129
+
130
+ BaseCallData::CapturedBatch& BaseCallData::CapturedBatch::operator=(
131
+ CapturedBatch&& b) noexcept {
132
+ Swap(&b);
133
+ return *this;
134
+ }
135
+
136
+ void BaseCallData::CapturedBatch::ResumeWith(Flusher* releaser) {
137
+ auto* batch = absl::exchange(batch_, nullptr);
138
+ GPR_ASSERT(batch != nullptr);
139
+ uintptr_t& refcnt = *RefCountField(batch);
140
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
141
+ if (--refcnt == 0) {
142
+ releaser->Resume(batch);
143
+ }
144
+ }
145
+
146
+ void BaseCallData::CapturedBatch::CompleteWith(Flusher* releaser) {
147
+ auto* batch = absl::exchange(batch_, nullptr);
148
+ GPR_ASSERT(batch != nullptr);
149
+ uintptr_t& refcnt = *RefCountField(batch);
150
+ if (refcnt == 0) return; // refcnt==0 ==> cancelled
151
+ if (--refcnt == 0) {
152
+ releaser->Complete(batch);
153
+ }
154
+ }
155
+
156
+ void BaseCallData::CapturedBatch::CancelWith(grpc_error_handle error,
157
+ Flusher* releaser) {
158
+ auto* batch = absl::exchange(batch_, nullptr);
159
+ GPR_ASSERT(batch != nullptr);
160
+ uintptr_t& refcnt = *RefCountField(batch);
161
+ if (refcnt == 0) {
162
+ // refcnt==0 ==> cancelled
163
+ GRPC_ERROR_UNREF(error);
164
+ return;
165
+ }
166
+ refcnt = 0;
167
+ releaser->Cancel(batch, error);
168
+ }
169
+
170
+ ///////////////////////////////////////////////////////////////////////////////
171
+ // BaseCallData::Flusher
172
+
173
+ BaseCallData::Flusher::Flusher(BaseCallData* call) : call_(call) {
174
+ GRPC_CALL_STACK_REF(call_->call_stack(), "flusher");
175
+ }
176
+
177
+ BaseCallData::Flusher::~Flusher() {
178
+ if (release_.empty()) {
179
+ if (call_closures_.size() == 0) {
180
+ GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "nothing to flush");
181
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
182
+ return;
183
+ }
184
+ call_closures_.RunClosures(call_->call_combiner());
185
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
186
+ return;
187
+ }
188
+ auto call_next_op = [](void* p, grpc_error_handle) {
189
+ auto* batch = static_cast<grpc_transport_stream_op_batch*>(p);
190
+ BaseCallData* call =
191
+ static_cast<BaseCallData*>(batch->handler_private.extra_arg);
192
+ grpc_call_next_op(call->elem(), batch);
193
+ GRPC_CALL_STACK_UNREF(call->call_stack(), "flusher_batch");
194
+ };
195
+ for (size_t i = 1; i < release_.size(); i++) {
196
+ auto* batch = release_[i];
197
+ batch->handler_private.extra_arg = call_;
198
+ GRPC_CLOSURE_INIT(&batch->handler_private.closure, call_next_op, batch,
199
+ nullptr);
200
+ GRPC_CALL_STACK_REF(call_->call_stack(), "flusher_batch");
201
+ call_closures_.Add(&batch->handler_private.closure, GRPC_ERROR_NONE,
202
+ "flusher_batch");
203
+ }
204
+ call_closures_.RunClosuresWithoutYielding(call_->call_combiner());
205
+ grpc_call_next_op(call_->elem(), release_[0]);
206
+ GRPC_CALL_STACK_UNREF(call_->call_stack(), "flusher");
207
+ }
208
+
73
209
  ///////////////////////////////////////////////////////////////////////////////
74
210
  // ClientCallData
75
211
 
@@ -104,8 +240,10 @@ struct ClientCallData::RecvInitialMetadata final {
104
240
 
105
241
  class ClientCallData::PollContext {
106
242
  public:
107
- explicit PollContext(ClientCallData* self) : self_(self) {
243
+ explicit PollContext(ClientCallData* self, Flusher* flusher)
244
+ : self_(self), flusher_(flusher) {
108
245
  GPR_ASSERT(self_->poll_ctx_ == nullptr);
246
+
109
247
  self_->poll_ctx_ = this;
110
248
  scoped_activity_.Init(self_);
111
249
  have_scoped_activity_ = true;
@@ -143,7 +281,7 @@ class ClientCallData::PollContext {
143
281
  }
144
282
  self_->recv_initial_metadata_->state =
145
283
  RecvInitialMetadata::kResponded;
146
- call_closures_.Add(
284
+ flusher_->AddClosure(
147
285
  absl::exchange(self_->recv_initial_metadata_->original_on_ready,
148
286
  nullptr),
149
287
  GRPC_ERROR_NONE,
@@ -171,7 +309,7 @@ class ClientCallData::PollContext {
171
309
  destroy_md = false;
172
310
  }
173
311
  self_->recv_trailing_state_ = RecvTrailingState::kResponded;
174
- call_closures_.Add(
312
+ flusher_->AddClosure(
175
313
  absl::exchange(self_->original_recv_trailing_metadata_ready_,
176
314
  nullptr),
177
315
  GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:1");
@@ -195,7 +333,7 @@ class ClientCallData::PollContext {
195
333
  case RecvInitialMetadata::kCompleteWaitingForLatch:
196
334
  self_->recv_initial_metadata_->state =
197
335
  RecvInitialMetadata::kResponded;
198
- call_closures_.Add(
336
+ flusher_->AddClosure(
199
337
  absl::exchange(
200
338
  self_->recv_initial_metadata_->original_on_ready,
201
339
  nullptr),
@@ -237,7 +375,7 @@ class ClientCallData::PollContext {
237
375
  case RecvInitialMetadata::kCompleteAndSetLatch:
238
376
  self_->recv_initial_metadata_->state =
239
377
  RecvInitialMetadata::kResponded;
240
- call_closures_.Add(
378
+ flusher_->AddClosure(
241
379
  absl::exchange(
242
380
  self_->recv_initial_metadata_->original_on_ready,
243
381
  nullptr),
@@ -247,21 +385,21 @@ class ClientCallData::PollContext {
247
385
  }
248
386
  if (self_->send_initial_state_ == SendInitialState::kQueued) {
249
387
  self_->send_initial_state_ = SendInitialState::kCancelled;
250
- cancel_send_initial_metadata_error_ = error;
388
+ self_->send_initial_metadata_batch_.CancelWith(error, flusher_);
251
389
  } else {
252
390
  GPR_ASSERT(
253
391
  self_->recv_trailing_state_ == RecvTrailingState::kInitial ||
254
392
  self_->recv_trailing_state_ == RecvTrailingState::kForwarded);
255
393
  self_->call_combiner()->Cancel(GRPC_ERROR_REF(error));
256
- forward_batch_ =
257
- grpc_make_transport_stream_op(GRPC_CLOSURE_CREATE(
258
- [](void* p, grpc_error_handle) {
259
- GRPC_CALL_COMBINER_STOP(static_cast<CallCombiner*>(p),
260
- "finish_cancel");
261
- },
262
- self_->call_combiner(), nullptr));
263
- forward_batch_->cancel_stream = true;
264
- forward_batch_->payload->cancel_stream.cancel_error = error;
394
+ CapturedBatch b(grpc_make_transport_stream_op(GRPC_CLOSURE_CREATE(
395
+ [](void* p, grpc_error_handle) {
396
+ GRPC_CALL_COMBINER_STOP(static_cast<CallCombiner*>(p),
397
+ "finish_cancel");
398
+ },
399
+ self_->call_combiner(), nullptr)));
400
+ b->cancel_stream = true;
401
+ b->payload->cancel_stream.cancel_error = error;
402
+ b.ResumeWith(flusher_);
265
403
  }
266
404
  self_->recv_trailing_state_ = RecvTrailingState::kCancelled;
267
405
  }
@@ -280,7 +418,7 @@ class ClientCallData::PollContext {
280
418
  // promise transition).
281
419
  if (self_->recv_trailing_state_ == RecvTrailingState::kComplete) {
282
420
  self_->recv_trailing_state_ = RecvTrailingState::kResponded;
283
- call_closures_.Add(
421
+ flusher_->AddClosure(
284
422
  absl::exchange(self_->original_recv_trailing_metadata_ready_,
285
423
  nullptr),
286
424
  GRPC_ERROR_NONE, "wake_inside_combiner:recv_trailing_ready:2");
@@ -292,77 +430,42 @@ class ClientCallData::PollContext {
292
430
  ~PollContext() {
293
431
  self_->poll_ctx_ = nullptr;
294
432
  if (have_scoped_activity_) scoped_activity_.Destroy();
295
- GRPC_CALL_STACK_REF(self_->call_stack(), "finish_poll");
296
- bool in_combiner = true;
297
- if (call_closures_.size() != 0) {
298
- if (forward_batch_ != nullptr) {
299
- call_closures_.RunClosuresWithoutYielding(self_->call_combiner());
300
- } else {
301
- in_combiner = false;
302
- call_closures_.RunClosures(self_->call_combiner());
303
- }
304
- }
305
- if (forward_batch_ != nullptr) {
306
- GPR_ASSERT(in_combiner);
307
- in_combiner = false;
308
- forward_send_initial_metadata_ = false;
309
- grpc_call_next_op(self_->elem(), forward_batch_);
310
- }
311
- if (cancel_send_initial_metadata_error_ != GRPC_ERROR_NONE) {
312
- GPR_ASSERT(in_combiner);
313
- forward_send_initial_metadata_ = false;
314
- in_combiner = false;
315
- grpc_transport_stream_op_batch_finish_with_failure(
316
- absl::exchange(self_->send_initial_metadata_batch_, nullptr),
317
- cancel_send_initial_metadata_error_, self_->call_combiner());
318
- }
319
- if (absl::exchange(forward_send_initial_metadata_, false)) {
320
- GPR_ASSERT(in_combiner);
321
- in_combiner = false;
322
- grpc_call_next_op(
323
- self_->elem(),
324
- absl::exchange(self_->send_initial_metadata_batch_, nullptr));
325
- }
326
433
  if (repoll_) {
327
- if (in_combiner) {
328
- self_->WakeInsideCombiner();
329
- } else {
330
- struct NextPoll : public grpc_closure {
331
- grpc_call_stack* call_stack;
332
- ClientCallData* call_data;
333
- };
334
- auto run = [](void* p, grpc_error_handle) {
335
- auto* next_poll = static_cast<NextPoll*>(p);
336
- next_poll->call_data->WakeInsideCombiner();
337
- GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll");
338
- delete next_poll;
339
- };
340
- auto* p = absl::make_unique<NextPoll>().release();
341
- p->call_stack = self_->call_stack();
342
- p->call_data = self_;
343
- GRPC_CALL_STACK_REF(self_->call_stack(), "re-poll");
344
- GRPC_CLOSURE_INIT(p, run, p, nullptr);
345
- GRPC_CALL_COMBINER_START(self_->call_combiner(), p, GRPC_ERROR_NONE,
346
- "re-poll");
347
- }
348
- } else if (in_combiner) {
349
- GRPC_CALL_COMBINER_STOP(self_->call_combiner(), "poll paused");
434
+ struct NextPoll : public grpc_closure {
435
+ grpc_call_stack* call_stack;
436
+ ClientCallData* call_data;
437
+ };
438
+ auto run = [](void* p, grpc_error_handle) {
439
+ auto* next_poll = static_cast<NextPoll*>(p);
440
+ {
441
+ Flusher flusher(next_poll->call_data);
442
+ next_poll->call_data->WakeInsideCombiner(&flusher);
443
+ }
444
+ GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll");
445
+ delete next_poll;
446
+ };
447
+ // Unique ptr --> release to suppress clang-tidy warnings about allocating
448
+ // in a destructor.
449
+ auto* p = absl::make_unique<NextPoll>().release();
450
+ p->call_stack = self_->call_stack();
451
+ p->call_data = self_;
452
+ GRPC_CALL_STACK_REF(self_->call_stack(), "re-poll");
453
+ GRPC_CLOSURE_INIT(p, run, p, nullptr);
454
+ flusher_->AddClosure(p, GRPC_ERROR_NONE, "re-poll");
350
455
  }
351
- GRPC_CALL_STACK_UNREF(self_->call_stack(), "finish_poll");
352
456
  }
353
457
 
354
458
  void Repoll() { repoll_ = true; }
355
459
 
356
- void ForwardSendInitialMetadata() { forward_send_initial_metadata_ = true; }
460
+ void ForwardSendInitialMetadata() {
461
+ self_->send_initial_metadata_batch_.ResumeWith(flusher_);
462
+ }
357
463
 
358
464
  private:
359
465
  ManualConstructor<ScopedActivity> scoped_activity_;
360
466
  ClientCallData* self_;
361
- CallCombinerClosureList call_closures_;
362
- grpc_error_handle cancel_send_initial_metadata_error_ = GRPC_ERROR_NONE;
363
- grpc_transport_stream_op_batch* forward_batch_ = nullptr;
467
+ Flusher* flusher_;
364
468
  bool repoll_ = false;
365
- bool forward_send_initial_metadata_ = false;
366
469
  bool have_scoped_activity_;
367
470
  };
368
471
 
@@ -393,9 +496,11 @@ void ClientCallData::ForceImmediateRepoll() {
393
496
  }
394
497
 
395
498
  // Handle one grpc_transport_stream_op_batch
396
- void ClientCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
499
+ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* b) {
397
500
  // Fake out the activity based context.
398
501
  ScopedContext context(this);
502
+ CapturedBatch batch(b);
503
+ Flusher flusher(this);
399
504
 
400
505
  // If this is a cancel stream, cancel anything we have pending and propagate
401
506
  // the cancellation.
@@ -405,7 +510,12 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
405
510
  !batch->recv_initial_metadata && !batch->recv_message &&
406
511
  !batch->recv_trailing_metadata);
407
512
  Cancel(batch->payload->cancel_stream.cancel_error);
408
- grpc_call_next_op(elem(), batch);
513
+ if (is_last()) {
514
+ GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
515
+ batch.CompleteWith(&flusher);
516
+ } else {
517
+ batch.ResumeWith(&flusher);
518
+ }
409
519
  return;
410
520
  }
411
521
 
@@ -451,40 +561,37 @@ void ClientCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
451
561
  // If we're already cancelled, just terminate the batch.
452
562
  if (send_initial_state_ == SendInitialState::kCancelled ||
453
563
  recv_trailing_state_ == RecvTrailingState::kCancelled) {
454
- grpc_transport_stream_op_batch_finish_with_failure(
455
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
456
- return;
457
- }
458
- // Otherwise, we should not have seen a send_initial_metadata op yet.
459
- GPR_ASSERT(send_initial_state_ == SendInitialState::kInitial);
460
- // Mark ourselves as queued.
461
- send_initial_state_ = SendInitialState::kQueued;
462
- if (batch->recv_trailing_metadata) {
463
- // If there's a recv_trailing_metadata op, we queue that too.
464
- GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
465
- recv_trailing_state_ = RecvTrailingState::kQueued;
564
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
565
+ } else {
566
+ // Otherwise, we should not have seen a send_initial_metadata op yet.
567
+ GPR_ASSERT(send_initial_state_ == SendInitialState::kInitial);
568
+ // Mark ourselves as queued.
569
+ send_initial_state_ = SendInitialState::kQueued;
570
+ if (batch->recv_trailing_metadata) {
571
+ // If there's a recv_trailing_metadata op, we queue that too.
572
+ GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
573
+ recv_trailing_state_ = RecvTrailingState::kQueued;
574
+ }
575
+ // This is the queuing!
576
+ send_initial_metadata_batch_ = batch;
577
+ // And kick start the promise.
578
+ StartPromise(&flusher);
466
579
  }
467
- // This is the queuing!
468
- send_initial_metadata_batch_ = batch;
469
- // And kick start the promise.
470
- StartPromise();
471
- return;
472
- }
473
-
474
- // recv_trailing_metadata *without* send_initial_metadata: hook it so we can
475
- // respond to it, and push it down.
476
- if (batch->recv_trailing_metadata) {
580
+ } else if (batch->recv_trailing_metadata) {
581
+ // recv_trailing_metadata *without* send_initial_metadata: hook it so we
582
+ // can respond to it, and push it down.
477
583
  if (recv_trailing_state_ == RecvTrailingState::kCancelled) {
478
- grpc_transport_stream_op_batch_finish_with_failure(
479
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
480
- return;
584
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
585
+ } else {
586
+ GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
587
+ recv_trailing_state_ = RecvTrailingState::kForwarded;
588
+ HookRecvTrailingMetadata(batch);
481
589
  }
482
- GPR_ASSERT(recv_trailing_state_ == RecvTrailingState::kInitial);
483
- recv_trailing_state_ = RecvTrailingState::kForwarded;
484
- HookRecvTrailingMetadata(batch);
590
+ } else if (cancelled_error_ != GRPC_ERROR_NONE) {
591
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
485
592
  }
486
593
 
487
- grpc_call_next_op(elem(), batch);
594
+ if (batch.is_captured()) batch.ResumeWith(&flusher);
488
595
  }
489
596
 
490
597
  // Handle cancellation.
@@ -502,19 +609,23 @@ void ClientCallData::Cancel(grpc_error_handle error) {
502
609
  recv_trailing_state_ = RecvTrailingState::kCancelled;
503
610
  }
504
611
  struct FailBatch : public grpc_closure {
505
- grpc_transport_stream_op_batch* batch;
506
- CallCombiner* call_combiner;
612
+ CapturedBatch batch;
613
+ ClientCallData* call;
507
614
  };
508
615
  auto fail = [](void* p, grpc_error_handle error) {
509
616
  auto* f = static_cast<FailBatch*>(p);
510
- grpc_transport_stream_op_batch_finish_with_failure(
511
- f->batch, GRPC_ERROR_REF(error), f->call_combiner);
617
+ {
618
+ Flusher flusher(f->call);
619
+ f->batch.CancelWith(GRPC_ERROR_REF(error), &flusher);
620
+ GRPC_CALL_STACK_UNREF(f->call->call_stack(), "cancel pending batch");
621
+ }
512
622
  delete f;
513
623
  };
514
624
  auto* b = new FailBatch();
515
625
  GRPC_CLOSURE_INIT(b, fail, b, nullptr);
516
- b->batch = absl::exchange(send_initial_metadata_batch_, nullptr);
517
- b->call_combiner = call_combiner();
626
+ b->batch = std::move(send_initial_metadata_batch_);
627
+ b->call = this;
628
+ GRPC_CALL_STACK_REF(call_stack(), "cancel pending batch");
518
629
  GRPC_CALL_COMBINER_START(call_combiner(), b,
519
630
  GRPC_ERROR_REF(cancelled_error_),
520
631
  "cancel pending batch");
@@ -545,12 +656,12 @@ void ClientCallData::Cancel(grpc_error_handle error) {
545
656
 
546
657
  // Begin running the promise - which will ultimately take some initial
547
658
  // metadata and return some trailing metadata.
548
- void ClientCallData::StartPromise() {
659
+ void ClientCallData::StartPromise(Flusher* flusher) {
549
660
  GPR_ASSERT(send_initial_state_ == SendInitialState::kQueued);
550
661
  ChannelFilter* filter = static_cast<ChannelFilter*>(elem()->channel_data);
551
662
 
552
663
  // Construct the promise.
553
- PollContext ctx(this);
664
+ PollContext ctx(this, flusher);
554
665
  promise_ = filter->MakeCallPromise(
555
666
  CallArgs{WrapMetadata(send_initial_metadata_batch_->payload
556
667
  ->send_initial_metadata.send_initial_metadata),
@@ -580,28 +691,26 @@ void ClientCallData::RecvInitialMetadataReady(grpc_error_handle error) {
580
691
  case RecvInitialMetadata::kRespondedToTrailingMetadataPriorToHook:
581
692
  abort(); // unreachable
582
693
  }
694
+ Flusher flusher(this);
583
695
  if (error != GRPC_ERROR_NONE) {
584
696
  recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
585
- GRPC_CALL_COMBINER_START(
586
- call_combiner(),
697
+ flusher.AddClosure(
587
698
  absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
588
699
  GRPC_ERROR_REF(error), "propagate cancellation");
589
700
  } else if (send_initial_state_ == SendInitialState::kCancelled ||
590
701
  recv_trailing_state_ == RecvTrailingState::kResponded) {
591
702
  recv_initial_metadata_->state = RecvInitialMetadata::kResponded;
592
- GRPC_CALL_COMBINER_START(
593
- call_combiner(),
703
+ flusher.AddClosure(
594
704
  absl::exchange(recv_initial_metadata_->original_on_ready, nullptr),
595
705
  GRPC_ERROR_REF(cancelled_error_), "propagate cancellation");
596
706
  }
597
- WakeInsideCombiner();
707
+ WakeInsideCombiner(&flusher);
598
708
  }
599
709
 
600
710
  // Interject our callback into the op batch for recv trailing metadata ready.
601
711
  // Stash a pointer to the trailing metadata that will be filled in, so we can
602
712
  // manipulate it later.
603
- void ClientCallData::HookRecvTrailingMetadata(
604
- grpc_transport_stream_op_batch* batch) {
713
+ void ClientCallData::HookRecvTrailingMetadata(CapturedBatch batch) {
605
714
  recv_trailing_metadata_ =
606
715
  batch->payload->recv_trailing_metadata.recv_trailing_metadata;
607
716
  original_recv_trailing_metadata_ready_ =
@@ -665,7 +774,7 @@ Poll<ServerMetadataHandle> ClientCallData::PollTrailingMetadata() {
665
774
  GPR_ASSERT(poll_ctx_ != nullptr);
666
775
  if (send_initial_state_ == SendInitialState::kQueued) {
667
776
  // First poll: pass the send_initial_metadata op down the stack.
668
- GPR_ASSERT(send_initial_metadata_batch_ != nullptr);
777
+ GPR_ASSERT(send_initial_metadata_batch_.is_captured());
669
778
  send_initial_state_ = SendInitialState::kForwarded;
670
779
  if (recv_trailing_state_ == RecvTrailingState::kQueued) {
671
780
  // (and the recv_trailing_metadata op if it's part of the queuing)
@@ -707,12 +816,14 @@ void ClientCallData::RecvTrailingMetadataReadyCallback(
707
816
  }
708
817
 
709
818
  void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
819
+ Flusher flusher(this);
710
820
  // If we were cancelled prior to receiving this callback, we should simply
711
821
  // forward the callback up with the same error.
712
822
  if (recv_trailing_state_ == RecvTrailingState::kCancelled) {
713
823
  if (grpc_closure* call_closure =
714
824
  absl::exchange(original_recv_trailing_metadata_ready_, nullptr)) {
715
- Closure::Run(DEBUG_LOCATION, call_closure, GRPC_ERROR_REF(error));
825
+ flusher.AddClosure(call_closure, GRPC_ERROR_REF(error),
826
+ "propagate failure");
716
827
  }
717
828
  return;
718
829
  }
@@ -726,7 +837,7 @@ void ClientCallData::RecvTrailingMetadataReady(grpc_error_handle error) {
726
837
  recv_trailing_state_ = RecvTrailingState::kComplete;
727
838
  // Repoll the promise.
728
839
  ScopedContext context(this);
729
- WakeInsideCombiner();
840
+ WakeInsideCombiner(&flusher);
730
841
  }
731
842
 
732
843
  // Given an error, fill in ServerMetadataHandle to represent that error.
@@ -743,37 +854,114 @@ void ClientCallData::SetStatusFromError(grpc_metadata_batch* metadata,
743
854
  }
744
855
 
745
856
  // Wakeup and poll the promise if appropriate.
746
- void ClientCallData::WakeInsideCombiner() { PollContext(this).Run(); }
857
+ void ClientCallData::WakeInsideCombiner(Flusher* flusher) {
858
+ PollContext(this, flusher).Run();
859
+ }
747
860
 
748
861
  void ClientCallData::OnWakeup() {
862
+ Flusher flusher(this);
749
863
  ScopedContext context(this);
750
- WakeInsideCombiner();
864
+ WakeInsideCombiner(&flusher);
751
865
  }
752
866
 
753
867
  ///////////////////////////////////////////////////////////////////////////////
754
868
  // ServerCallData
755
869
 
870
+ struct ServerCallData::SendInitialMetadata {
871
+ enum State {
872
+ kInitial,
873
+ kGotLatch,
874
+ kQueuedWaitingForLatch,
875
+ kQueuedAndGotLatch,
876
+ kQueuedAndSetLatch,
877
+ kForwarded,
878
+ kCancelled,
879
+ };
880
+ State state = kInitial;
881
+ CapturedBatch batch;
882
+ Latch<ServerMetadata*>* server_initial_metadata_publisher = nullptr;
883
+ };
884
+
885
+ class ServerCallData::PollContext {
886
+ public:
887
+ explicit PollContext(ServerCallData* self, Flusher* flusher)
888
+ : self_(self), flusher_(flusher) {
889
+ GPR_ASSERT(self_->poll_ctx_ == nullptr);
890
+ self_->poll_ctx_ = this;
891
+ scoped_activity_.Init(self_);
892
+ have_scoped_activity_ = true;
893
+ }
894
+
895
+ PollContext(const PollContext&) = delete;
896
+ PollContext& operator=(const PollContext&) = delete;
897
+
898
+ ~PollContext() {
899
+ self_->poll_ctx_ = nullptr;
900
+ if (have_scoped_activity_) scoped_activity_.Destroy();
901
+ if (repoll_) {
902
+ struct NextPoll : public grpc_closure {
903
+ grpc_call_stack* call_stack;
904
+ ServerCallData* call_data;
905
+ };
906
+ auto run = [](void* p, grpc_error_handle) {
907
+ auto* next_poll = static_cast<NextPoll*>(p);
908
+ {
909
+ Flusher flusher(next_poll->call_data);
910
+ next_poll->call_data->WakeInsideCombiner(&flusher);
911
+ }
912
+ GRPC_CALL_STACK_UNREF(next_poll->call_stack, "re-poll");
913
+ delete next_poll;
914
+ };
915
+ auto* p = absl::make_unique<NextPoll>().release();
916
+ p->call_stack = self_->call_stack();
917
+ p->call_data = self_;
918
+ GRPC_CALL_STACK_REF(self_->call_stack(), "re-poll");
919
+ GRPC_CLOSURE_INIT(p, run, p, nullptr);
920
+ flusher_->AddClosure(p, GRPC_ERROR_NONE, "re-poll");
921
+ }
922
+ }
923
+
924
+ void Repoll() { repoll_ = true; }
925
+ void ClearRepoll() { repoll_ = false; }
926
+
927
+ private:
928
+ ManualConstructor<ScopedActivity> scoped_activity_;
929
+ ServerCallData* const self_;
930
+ Flusher* const flusher_;
931
+ bool repoll_ = false;
932
+ bool have_scoped_activity_;
933
+ };
934
+
756
935
  ServerCallData::ServerCallData(grpc_call_element* elem,
757
936
  const grpc_call_element_args* args,
758
937
  uint8_t flags)
759
938
  : BaseCallData(elem, args, flags) {
939
+ if (server_initial_metadata_latch() != nullptr) {
940
+ send_initial_metadata_ = arena()->New<SendInitialMetadata>();
941
+ }
760
942
  GRPC_CLOSURE_INIT(&recv_initial_metadata_ready_,
761
943
  RecvInitialMetadataReadyCallback, this,
762
944
  grpc_schedule_on_exec_ctx);
763
945
  }
764
946
 
765
947
  ServerCallData::~ServerCallData() {
766
- GPR_ASSERT(!is_polling_);
948
+ GPR_ASSERT(poll_ctx_ == nullptr);
767
949
  GRPC_ERROR_UNREF(cancelled_error_);
768
950
  }
769
951
 
770
952
  // Activity implementation.
771
- void ServerCallData::ForceImmediateRepoll() { abort(); } // Not implemented.
953
+ void ServerCallData::ForceImmediateRepoll() {
954
+ GPR_ASSERT(poll_ctx_ != nullptr);
955
+ poll_ctx_->Repoll();
956
+ }
772
957
 
773
958
  // Handle one grpc_transport_stream_op_batch
774
- void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
959
+ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* b) {
775
960
  // Fake out the activity based context.
776
961
  ScopedContext context(this);
962
+ CapturedBatch batch(b);
963
+ Flusher flusher(this);
964
+ bool wake = false;
777
965
 
778
966
  // If this is a cancel stream, cancel anything we have pending and
779
967
  // propagate the cancellation.
@@ -782,8 +970,14 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
782
970
  !batch->send_trailing_metadata && !batch->send_message &&
783
971
  !batch->recv_initial_metadata && !batch->recv_message &&
784
972
  !batch->recv_trailing_metadata);
785
- Cancel(batch->payload->cancel_stream.cancel_error);
786
- grpc_call_next_op(elem(), batch);
973
+ Cancel(GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error),
974
+ &flusher);
975
+ if (is_last()) {
976
+ GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error);
977
+ batch.CompleteWith(&flusher);
978
+ } else {
979
+ batch.ResumeWith(&flusher);
980
+ }
787
981
  return;
788
982
  }
789
983
 
@@ -805,61 +999,85 @@ void ServerCallData::StartBatch(grpc_transport_stream_op_batch* batch) {
805
999
  recv_initial_state_ = RecvInitialState::kForwarded;
806
1000
  }
807
1001
 
1002
+ // send_initial_metadata
1003
+ if (send_initial_metadata_ != nullptr && batch->send_initial_metadata) {
1004
+ switch (send_initial_metadata_->state) {
1005
+ case SendInitialMetadata::kInitial:
1006
+ send_initial_metadata_->state =
1007
+ SendInitialMetadata::kQueuedWaitingForLatch;
1008
+ break;
1009
+ case SendInitialMetadata::kGotLatch:
1010
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch;
1011
+ break;
1012
+ case SendInitialMetadata::kCancelled:
1013
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
1014
+ break;
1015
+ case SendInitialMetadata::kQueuedAndGotLatch:
1016
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1017
+ case SendInitialMetadata::kQueuedAndSetLatch:
1018
+ case SendInitialMetadata::kForwarded:
1019
+ abort(); // not reachable
1020
+ }
1021
+ send_initial_metadata_->batch = batch;
1022
+ wake = true;
1023
+ }
1024
+
808
1025
  // send_trailing_metadata
809
- if (batch->send_trailing_metadata) {
1026
+ if (batch.is_captured() && batch->send_trailing_metadata) {
810
1027
  switch (send_trailing_state_) {
811
1028
  case SendTrailingState::kInitial:
812
1029
  send_trailing_metadata_batch_ = batch;
813
1030
  send_trailing_state_ = SendTrailingState::kQueued;
814
- WakeInsideCombiner([this](grpc_error_handle error) {
815
- GPR_ASSERT(send_trailing_state_ == SendTrailingState::kQueued);
816
- Cancel(error);
817
- });
1031
+ wake = true;
818
1032
  break;
819
1033
  case SendTrailingState::kQueued:
820
1034
  case SendTrailingState::kForwarded:
821
1035
  abort(); // unreachable
822
1036
  break;
823
1037
  case SendTrailingState::kCancelled:
824
- grpc_transport_stream_op_batch_finish_with_failure(
825
- batch, GRPC_ERROR_REF(cancelled_error_), call_combiner());
1038
+ batch.CancelWith(GRPC_ERROR_REF(cancelled_error_), &flusher);
826
1039
  break;
827
1040
  }
828
- return;
829
1041
  }
830
1042
 
831
- grpc_call_next_op(elem(), batch);
1043
+ if (wake) WakeInsideCombiner(&flusher);
1044
+ if (batch.is_captured()) batch.ResumeWith(&flusher);
832
1045
  }
833
1046
 
834
1047
  // Handle cancellation.
835
- void ServerCallData::Cancel(grpc_error_handle error) {
1048
+ void ServerCallData::Cancel(grpc_error_handle error, Flusher* flusher) {
836
1049
  // Track the latest reason for cancellation.
837
1050
  GRPC_ERROR_UNREF(cancelled_error_);
838
- cancelled_error_ = GRPC_ERROR_REF(error);
1051
+ cancelled_error_ = error;
839
1052
  // Stop running the promise.
840
1053
  promise_ = ArenaPromise<ServerMetadataHandle>();
841
1054
  if (send_trailing_state_ == SendTrailingState::kQueued) {
842
1055
  send_trailing_state_ = SendTrailingState::kCancelled;
843
- struct FailBatch : public grpc_closure {
844
- grpc_transport_stream_op_batch* batch;
845
- CallCombiner* call_combiner;
846
- };
847
- auto fail = [](void* p, grpc_error_handle error) {
848
- auto* f = static_cast<FailBatch*>(p);
849
- grpc_transport_stream_op_batch_finish_with_failure(
850
- f->batch, GRPC_ERROR_REF(error), f->call_combiner);
851
- delete f;
852
- };
853
- auto* b = new FailBatch();
854
- GRPC_CLOSURE_INIT(b, fail, b, nullptr);
855
- b->batch = absl::exchange(send_trailing_metadata_batch_, nullptr);
856
- b->call_combiner = call_combiner();
857
- GRPC_CALL_COMBINER_START(call_combiner(), b,
858
- GRPC_ERROR_REF(cancelled_error_),
859
- "cancel pending batch");
1056
+ send_trailing_metadata_batch_.CancelWith(GRPC_ERROR_REF(error), flusher);
860
1057
  } else {
861
1058
  send_trailing_state_ = SendTrailingState::kCancelled;
862
1059
  }
1060
+ if (send_initial_metadata_ != nullptr) {
1061
+ switch (send_initial_metadata_->state) {
1062
+ case SendInitialMetadata::kInitial:
1063
+ case SendInitialMetadata::kGotLatch:
1064
+ case SendInitialMetadata::kForwarded:
1065
+ case SendInitialMetadata::kCancelled:
1066
+ break;
1067
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1068
+ case SendInitialMetadata::kQueuedAndGotLatch:
1069
+ case SendInitialMetadata::kQueuedAndSetLatch:
1070
+ send_initial_metadata_->batch.CancelWith(GRPC_ERROR_REF(error),
1071
+ flusher);
1072
+ break;
1073
+ }
1074
+ send_initial_metadata_->state = SendInitialMetadata::kCancelled;
1075
+ }
1076
+ if (auto* closure =
1077
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
1078
+ flusher->AddClosure(closure, GRPC_ERROR_REF(error),
1079
+ "original_recv_initial_metadata");
1080
+ }
863
1081
  }
864
1082
 
865
1083
  // Construct a promise that will "call" the next filter.
@@ -872,6 +1090,31 @@ ArenaPromise<ServerMetadataHandle> ServerCallData::MakeNextPromise(
872
1090
  GPR_ASSERT(UnwrapMetadata(std::move(call_args.client_initial_metadata)) ==
873
1091
  recv_initial_metadata_);
874
1092
  forward_recv_initial_metadata_callback_ = true;
1093
+ if (send_initial_metadata_ != nullptr) {
1094
+ GPR_ASSERT(send_initial_metadata_->server_initial_metadata_publisher ==
1095
+ nullptr);
1096
+ GPR_ASSERT(call_args.server_initial_metadata != nullptr);
1097
+ send_initial_metadata_->server_initial_metadata_publisher =
1098
+ call_args.server_initial_metadata;
1099
+ switch (send_initial_metadata_->state) {
1100
+ case SendInitialMetadata::kInitial:
1101
+ send_initial_metadata_->state = SendInitialMetadata::kGotLatch;
1102
+ break;
1103
+ case SendInitialMetadata::kGotLatch:
1104
+ case SendInitialMetadata::kQueuedAndGotLatch:
1105
+ case SendInitialMetadata::kQueuedAndSetLatch:
1106
+ case SendInitialMetadata::kForwarded:
1107
+ abort(); // not reachable
1108
+ break;
1109
+ case SendInitialMetadata::kQueuedWaitingForLatch:
1110
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndGotLatch;
1111
+ break;
1112
+ case SendInitialMetadata::kCancelled:
1113
+ break;
1114
+ }
1115
+ } else {
1116
+ GPR_ASSERT(call_args.server_initial_metadata == nullptr);
1117
+ }
875
1118
  return ArenaPromise<ServerMetadataHandle>(
876
1119
  [this]() { return PollTrailingMetadata(); });
877
1120
  }
@@ -903,12 +1146,14 @@ void ServerCallData::RecvInitialMetadataReadyCallback(void* arg,
903
1146
  }
904
1147
 
905
1148
  void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
1149
+ Flusher flusher(this);
906
1150
  GPR_ASSERT(recv_initial_state_ == RecvInitialState::kForwarded);
907
1151
  // If there was an error we just propagate that through
908
1152
  if (error != GRPC_ERROR_NONE) {
909
1153
  recv_initial_state_ = RecvInitialState::kResponded;
910
- Closure::Run(DEBUG_LOCATION, original_recv_initial_metadata_ready_,
911
- GRPC_ERROR_REF(error));
1154
+ flusher.AddClosure(
1155
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr),
1156
+ GRPC_ERROR_REF(error), "propagate error");
912
1157
  return;
913
1158
  }
914
1159
  // Record that we've got the callback.
@@ -925,30 +1170,46 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) {
925
1170
  return MakeNextPromise(std::move(call_args));
926
1171
  });
927
1172
  // Poll once.
928
- bool own_error = false;
929
- WakeInsideCombiner([&error, &own_error](grpc_error_handle new_error) {
930
- GPR_ASSERT(error == GRPC_ERROR_NONE);
931
- error = GRPC_ERROR_REF(new_error);
932
- own_error = true;
933
- });
934
- Closure::Run(DEBUG_LOCATION, original_recv_initial_metadata_ready_,
935
- GRPC_ERROR_REF(error));
936
- if (own_error) GRPC_ERROR_UNREF(error);
1173
+ WakeInsideCombiner(&flusher);
1174
+ if (auto* closure =
1175
+ absl::exchange(original_recv_initial_metadata_ready_, nullptr)) {
1176
+ flusher.AddClosure(closure, GRPC_ERROR_NONE,
1177
+ "original_recv_initial_metadata");
1178
+ }
937
1179
  }
938
1180
 
939
1181
  // Wakeup and poll the promise if appropriate.
940
- void ServerCallData::WakeInsideCombiner(
941
- absl::FunctionRef<void(grpc_error_handle)> cancel) {
942
- GPR_ASSERT(!is_polling_);
943
- bool forward_send_trailing_metadata = false;
944
- is_polling_ = true;
945
- if (recv_initial_state_ == RecvInitialState::kComplete) {
1182
+ void ServerCallData::WakeInsideCombiner(Flusher* flusher) {
1183
+ PollContext poll_ctx(this, flusher);
1184
+ if (send_initial_metadata_ != nullptr &&
1185
+ send_initial_metadata_->state ==
1186
+ SendInitialMetadata::kQueuedAndGotLatch) {
1187
+ send_initial_metadata_->state = SendInitialMetadata::kQueuedAndSetLatch;
1188
+ send_initial_metadata_->server_initial_metadata_publisher->Set(
1189
+ send_initial_metadata_->batch->payload->send_initial_metadata
1190
+ .send_initial_metadata);
1191
+ }
1192
+ poll_ctx.ClearRepoll();
1193
+ if (promise_.has_value()) {
946
1194
  Poll<ServerMetadataHandle> poll;
947
- {
948
- ScopedActivity activity(this);
949
- poll = promise_();
1195
+ poll = promise_();
1196
+ if (send_initial_metadata_ != nullptr &&
1197
+ send_initial_metadata_->state ==
1198
+ SendInitialMetadata::kQueuedAndSetLatch) {
1199
+ Poll<ServerMetadata**> p = server_initial_metadata_latch()->Wait()();
1200
+ if (ServerMetadata*** ppp = absl::get_if<ServerMetadata**>(&p)) {
1201
+ ServerMetadata* md = **ppp;
1202
+ if (send_initial_metadata_->batch->payload->send_initial_metadata
1203
+ .send_initial_metadata != md) {
1204
+ *send_initial_metadata_->batch->payload->send_initial_metadata
1205
+ .send_initial_metadata = std::move(*md);
1206
+ }
1207
+ send_initial_metadata_->state = SendInitialMetadata::kForwarded;
1208
+ send_initial_metadata_->batch.ResumeWith(flusher);
1209
+ }
950
1210
  }
951
1211
  if (auto* r = absl::get_if<ServerMetadataHandle>(&poll)) {
1212
+ promise_ = ArenaPromise<ServerMetadataHandle>();
952
1213
  auto* md = UnwrapMetadata(std::move(*r));
953
1214
  bool destroy_md = true;
954
1215
  switch (send_trailing_state_) {
@@ -960,7 +1221,7 @@ void ServerCallData::WakeInsideCombiner(
960
1221
  } else {
961
1222
  destroy_md = false;
962
1223
  }
963
- forward_send_trailing_metadata = true;
1224
+ send_trailing_metadata_batch_.ResumeWith(flusher);
964
1225
  send_trailing_state_ = SendTrailingState::kForwarded;
965
1226
  } break;
966
1227
  case SendTrailingState::kForwarded:
@@ -977,8 +1238,7 @@ void ServerCallData::WakeInsideCombiner(
977
1238
  error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE,
978
1239
  message->as_string_view());
979
1240
  }
980
- cancel(error);
981
- GRPC_ERROR_UNREF(error);
1241
+ Cancel(error, flusher);
982
1242
  } break;
983
1243
  case SendTrailingState::kCancelled:
984
1244
  // Nothing to do.
@@ -989,11 +1249,6 @@ void ServerCallData::WakeInsideCombiner(
989
1249
  }
990
1250
  }
991
1251
  }
992
- is_polling_ = false;
993
- if (forward_send_trailing_metadata) {
994
- grpc_call_next_op(elem(),
995
- absl::exchange(send_trailing_metadata_batch_, nullptr));
996
- }
997
1252
  }
998
1253
 
999
1254
  void ServerCallData::OnWakeup() { abort(); } // not implemented