grpc 1.64.0 → 1.65.0.pre1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (622) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +57 -58
  3. data/include/grpc/event_engine/event_engine.h +13 -6
  4. data/include/grpc/impl/channel_arg_names.h +7 -3
  5. data/include/grpc/module.modulemap +1 -0
  6. data/include/grpc/passive_listener.h +62 -0
  7. data/include/grpc/support/log.h +1 -6
  8. data/include/grpc/support/port_platform.h +3 -0
  9. data/src/core/channelz/channel_trace.cc +1 -1
  10. data/src/core/channelz/channel_trace.h +1 -1
  11. data/src/core/channelz/channelz.cc +3 -3
  12. data/src/core/channelz/channelz.h +7 -7
  13. data/src/core/channelz/channelz_registry.cc +4 -3
  14. data/src/core/client_channel/backup_poller.cc +4 -5
  15. data/src/core/client_channel/client_channel.cc +1324 -0
  16. data/src/core/client_channel/client_channel.h +243 -0
  17. data/src/core/client_channel/client_channel_filter.cc +266 -709
  18. data/src/core/client_channel/client_channel_filter.h +11 -64
  19. data/src/core/client_channel/client_channel_internal.h +16 -5
  20. data/src/core/client_channel/client_channel_plugin.cc +1 -14
  21. data/src/core/client_channel/client_channel_service_config.h +3 -3
  22. data/src/core/client_channel/config_selector.cc +1 -1
  23. data/src/core/client_channel/config_selector.h +1 -1
  24. data/src/core/client_channel/dynamic_filters.cc +3 -3
  25. data/src/core/client_channel/dynamic_filters.h +1 -3
  26. data/src/core/client_channel/load_balanced_call_destination.cc +336 -0
  27. data/src/core/client_channel/load_balanced_call_destination.h +49 -0
  28. data/src/core/client_channel/retry_filter.cc +2 -9
  29. data/src/core/client_channel/retry_filter.h +2 -7
  30. data/src/core/client_channel/retry_filter_legacy_call_data.cc +65 -72
  31. data/src/core/client_channel/retry_filter_legacy_call_data.h +0 -2
  32. data/src/core/client_channel/retry_service_config.cc +4 -5
  33. data/src/core/client_channel/retry_service_config.h +3 -3
  34. data/src/core/client_channel/subchannel.cc +220 -112
  35. data/src/core/client_channel/subchannel.h +31 -18
  36. data/src/core/client_channel/subchannel_pool_interface.cc +0 -2
  37. data/src/core/client_channel/subchannel_pool_interface.h +2 -4
  38. data/src/core/client_channel/subchannel_stream_client.cc +36 -49
  39. data/src/core/client_channel/subchannel_stream_client.h +2 -4
  40. data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +7 -10
  41. data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +1 -0
  42. data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +7 -0
  43. data/src/core/ext/filters/census/grpc_context.cc +2 -4
  44. data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +8 -15
  45. data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h +2 -0
  46. data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +3 -7
  47. data/src/core/ext/filters/fault_injection/fault_injection_filter.h +1 -0
  48. data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h +3 -3
  49. data/src/core/ext/filters/http/client/http_client_filter.cc +1 -0
  50. data/src/core/ext/filters/http/client/http_client_filter.h +1 -0
  51. data/src/core/ext/filters/http/client_authority_filter.cc +1 -0
  52. data/src/core/ext/filters/http/client_authority_filter.h +1 -0
  53. data/src/core/ext/filters/http/message_compress/compression_filter.cc +10 -15
  54. data/src/core/ext/filters/http/message_compress/compression_filter.h +2 -0
  55. data/src/core/ext/filters/http/server/http_server_filter.cc +2 -2
  56. data/src/core/ext/filters/http/server/http_server_filter.h +1 -0
  57. data/src/core/ext/filters/message_size/message_size_filter.cc +6 -9
  58. data/src/core/ext/filters/message_size/message_size_filter.h +6 -6
  59. data/src/core/ext/filters/rbac/rbac_filter.cc +2 -5
  60. data/src/core/ext/filters/rbac/rbac_filter.h +1 -0
  61. data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +2 -2
  62. data/src/core/ext/filters/rbac/rbac_service_config_parser.h +1 -1
  63. data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +2 -6
  64. data/src/core/ext/filters/stateful_session/stateful_session_filter.h +1 -0
  65. data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +3 -3
  66. data/src/core/ext/transport/chttp2/alpn/alpn.cc +1 -1
  67. data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +8 -25
  68. data/src/core/ext/transport/chttp2/client/chttp2_connector.h +0 -5
  69. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +223 -148
  70. data/src/core/ext/transport/chttp2/server/chttp2_server.h +33 -0
  71. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +131 -107
  72. data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +8 -8
  73. data/src/core/ext/transport/chttp2/transport/context_list_entry.h +1 -1
  74. data/src/core/ext/transport/chttp2/transport/flow_control.cc +2 -4
  75. data/src/core/ext/transport/chttp2/transport/flow_control.h +0 -2
  76. data/src/core/ext/transport/chttp2/transport/frame_ping.cc +4 -6
  77. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +1 -2
  78. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +9 -5
  79. data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +5 -4
  80. data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +9 -2
  81. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +5 -7
  82. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +1 -1
  83. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +3 -8
  84. data/src/core/ext/transport/chttp2/transport/http2_settings.cc +1 -1
  85. data/src/core/ext/transport/chttp2/transport/http2_settings.h +1 -1
  86. data/src/core/ext/transport/chttp2/transport/internal.h +29 -19
  87. data/src/core/ext/transport/chttp2/transport/parsing.cc +15 -25
  88. data/src/core/ext/transport/chttp2/transport/ping_callbacks.cc +0 -2
  89. data/src/core/ext/transport/chttp2/transport/ping_callbacks.h +0 -2
  90. data/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc +29 -13
  91. data/src/core/ext/transport/chttp2/transport/ping_rate_policy.h +5 -4
  92. data/src/core/ext/transport/chttp2/transport/stream_lists.cc +3 -5
  93. data/src/core/ext/transport/chttp2/transport/writing.cc +24 -25
  94. data/src/core/ext/transport/inproc/inproc_transport.cc +56 -32
  95. data/src/core/ext/transport/inproc/inproc_transport.h +1 -3
  96. data/src/core/ext/transport/inproc/legacy_inproc_transport.cc +13 -15
  97. data/src/core/ext/transport/inproc/legacy_inproc_transport.h +0 -2
  98. data/src/core/handshaker/handshaker.cc +6 -14
  99. data/src/core/handshaker/http_connect/http_connect_handshaker.cc +9 -17
  100. data/src/core/handshaker/http_connect/http_proxy_mapper.cc +3 -2
  101. data/src/core/handshaker/security/secure_endpoint.cc +38 -32
  102. data/src/core/handshaker/security/secure_endpoint.h +0 -2
  103. data/src/core/handshaker/security/security_handshaker.cc +25 -37
  104. data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc +2 -1
  105. data/src/core/lib/address_utils/parse_address.cc +27 -39
  106. data/src/core/lib/address_utils/sockaddr_utils.cc +5 -6
  107. data/src/core/lib/avl/avl.h +1 -1
  108. data/src/core/lib/channel/channel_args.cc +13 -17
  109. data/src/core/lib/channel/channel_args.h +19 -8
  110. data/src/core/lib/channel/channel_stack.cc +5 -63
  111. data/src/core/lib/channel/channel_stack.h +13 -37
  112. data/src/core/lib/channel/channel_stack_builder.h +0 -5
  113. data/src/core/lib/channel/channel_stack_builder_impl.cc +0 -142
  114. data/src/core/lib/channel/channel_stack_builder_impl.h +0 -2
  115. data/src/core/lib/channel/connected_channel.cc +37 -676
  116. data/src/core/lib/channel/promise_based_filter.cc +41 -47
  117. data/src/core/lib/channel/promise_based_filter.h +124 -477
  118. data/src/core/lib/channel/status_util.cc +1 -1
  119. data/src/core/lib/compression/compression.cc +1 -1
  120. data/src/core/lib/compression/message_compress.cc +6 -6
  121. data/src/core/lib/config/config_vars.cc +2 -7
  122. data/src/core/lib/config/config_vars.h +1 -5
  123. data/src/core/lib/debug/event_log.h +1 -1
  124. data/src/core/lib/debug/trace.cc +43 -59
  125. data/src/core/lib/debug/trace.h +2 -97
  126. data/src/core/lib/debug/trace_flags.cc +255 -0
  127. data/src/core/lib/debug/trace_flags.h +133 -0
  128. data/src/core/lib/debug/trace_impl.h +115 -0
  129. data/src/core/lib/event_engine/ares_resolver.cc +5 -7
  130. data/src/core/lib/event_engine/ares_resolver.h +1 -3
  131. data/src/core/lib/event_engine/cf_engine/cf_engine.cc +1 -1
  132. data/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc +17 -22
  133. data/src/core/lib/event_engine/event_engine.cc +29 -4
  134. data/src/core/lib/event_engine/extensions/supports_fd.h +7 -0
  135. data/src/core/lib/event_engine/extensions/tcp_trace.h +43 -0
  136. data/src/core/lib/event_engine/forkable.cc +4 -5
  137. data/src/core/lib/event_engine/forkable.h +0 -11
  138. data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +10 -11
  139. data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +4 -3
  140. data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc +1 -1
  141. data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +19 -33
  142. data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +3 -2
  143. data/src/core/lib/event_engine/posix_engine/posix_engine.cc +24 -7
  144. data/src/core/lib/event_engine/posix_engine/posix_engine.h +2 -0
  145. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +14 -16
  146. data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +18 -22
  147. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +13 -17
  148. data/src/core/lib/event_engine/posix_engine/timer.cc +1 -1
  149. data/src/core/lib/event_engine/posix_engine/timer_manager.cc +4 -6
  150. data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +2 -1
  151. data/src/core/lib/event_engine/shim.cc +1 -1
  152. data/src/core/lib/event_engine/tcp_socket_utils.cc +6 -8
  153. data/src/core/lib/event_engine/thread_local.h +1 -1
  154. data/src/core/lib/event_engine/thread_pool/thread_count.h +1 -1
  155. data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc +19 -21
  156. data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h +3 -6
  157. data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc +14 -13
  158. data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h +4 -3
  159. data/src/core/lib/event_engine/trace.h +6 -17
  160. data/src/core/lib/event_engine/windows/iocp.h +1 -1
  161. data/src/core/lib/event_engine/windows/win_socket.cc +23 -17
  162. data/src/core/lib/event_engine/windows/win_socket.h +4 -5
  163. data/src/core/lib/event_engine/windows/windows_endpoint.cc +6 -9
  164. data/src/core/lib/event_engine/windows/windows_engine.cc +201 -87
  165. data/src/core/lib/event_engine/windows/windows_engine.h +136 -25
  166. data/src/core/lib/event_engine/windows/windows_listener.cc +12 -23
  167. data/src/core/lib/experiments/experiments.cc +35 -151
  168. data/src/core/lib/experiments/experiments.h +12 -45
  169. data/src/core/lib/gprpp/bitset.h +1 -1
  170. data/src/core/lib/gprpp/crash.cc +2 -3
  171. data/src/core/lib/gprpp/dual_ref_counted.h +45 -33
  172. data/src/core/lib/gprpp/dump_args.cc +54 -0
  173. data/src/core/lib/gprpp/dump_args.h +69 -0
  174. data/src/core/lib/gprpp/glob.cc +70 -0
  175. data/src/core/lib/gprpp/glob.h +29 -0
  176. data/src/core/lib/gprpp/per_cpu.cc +1 -1
  177. data/src/core/lib/gprpp/posix/stat.cc +3 -4
  178. data/src/core/lib/gprpp/posix/thd.cc +8 -9
  179. data/src/core/lib/gprpp/ref_counted.h +30 -22
  180. data/src/core/lib/gprpp/single_set_ptr.h +5 -3
  181. data/src/core/lib/gprpp/status_helper.cc +11 -30
  182. data/src/core/lib/gprpp/status_helper.h +3 -31
  183. data/src/core/lib/gprpp/time.cc +3 -4
  184. data/src/core/lib/gprpp/time.h +3 -2
  185. data/src/core/lib/gprpp/unique_type_name.h +1 -1
  186. data/src/core/lib/gprpp/validation_errors.cc +10 -1
  187. data/src/core/lib/gprpp/validation_errors.h +11 -0
  188. data/src/core/lib/gprpp/windows/stat.cc +3 -4
  189. data/src/core/lib/gprpp/windows/thd.cc +3 -2
  190. data/src/core/lib/gprpp/work_serializer.cc +48 -57
  191. data/src/core/lib/iomgr/buffer_list.cc +4 -2
  192. data/src/core/lib/iomgr/call_combiner.cc +18 -27
  193. data/src/core/lib/iomgr/call_combiner.h +1 -3
  194. data/src/core/lib/iomgr/cfstream_handle.cc +4 -6
  195. data/src/core/lib/iomgr/closure.h +2 -4
  196. data/src/core/lib/iomgr/combiner.cc +6 -8
  197. data/src/core/lib/iomgr/combiner.h +0 -2
  198. data/src/core/lib/iomgr/endpoint.cc +0 -6
  199. data/src/core/lib/iomgr/endpoint.h +0 -2
  200. data/src/core/lib/iomgr/endpoint_cfstream.cc +19 -41
  201. data/src/core/lib/iomgr/endpoint_pair_posix.cc +1 -1
  202. data/src/core/lib/iomgr/endpoint_pair_windows.cc +5 -6
  203. data/src/core/lib/iomgr/error.cc +13 -21
  204. data/src/core/lib/iomgr/error.h +1 -1
  205. data/src/core/lib/iomgr/ev_apple.cc +3 -5
  206. data/src/core/lib/iomgr/ev_epoll1_linux.cc +43 -42
  207. data/src/core/lib/iomgr/ev_poll_posix.cc +38 -29
  208. data/src/core/lib/iomgr/ev_posix.cc +8 -9
  209. data/src/core/lib/iomgr/ev_posix.h +10 -7
  210. data/src/core/lib/iomgr/event_engine_shims/closure.cc +2 -2
  211. data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +14 -28
  212. data/src/core/lib/iomgr/exec_ctx.cc +2 -2
  213. data/src/core/lib/iomgr/exec_ctx.h +1 -1
  214. data/src/core/lib/iomgr/executor.cc +6 -15
  215. data/src/core/lib/iomgr/executor.h +1 -1
  216. data/src/core/lib/iomgr/fork_posix.cc +8 -10
  217. data/src/core/lib/iomgr/fork_windows.cc +3 -1
  218. data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +2 -3
  219. data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +3 -5
  220. data/src/core/lib/iomgr/internal_errqueue.cc +4 -2
  221. data/src/core/lib/iomgr/iocp_windows.cc +4 -3
  222. data/src/core/lib/iomgr/iomgr.cc +13 -17
  223. data/src/core/lib/iomgr/lockfree_event.cc +3 -5
  224. data/src/core/lib/iomgr/pollset.h +0 -2
  225. data/src/core/lib/iomgr/pollset_windows.cc +0 -2
  226. data/src/core/lib/iomgr/resolve_address_posix.cc +7 -14
  227. data/src/core/lib/iomgr/resolve_address_windows.cc +1 -1
  228. data/src/core/lib/iomgr/socket_factory_posix.cc +1 -1
  229. data/src/core/lib/iomgr/socket_mutator.cc +1 -1
  230. data/src/core/lib/iomgr/socket_utils_common_posix.cc +11 -17
  231. data/src/core/lib/iomgr/socket_windows.cc +4 -6
  232. data/src/core/lib/iomgr/tcp_client_cfstream.cc +3 -5
  233. data/src/core/lib/iomgr/tcp_client_posix.cc +9 -15
  234. data/src/core/lib/iomgr/tcp_client_windows.cc +2 -4
  235. data/src/core/lib/iomgr/tcp_posix.cc +57 -84
  236. data/src/core/lib/iomgr/tcp_posix.h +0 -2
  237. data/src/core/lib/iomgr/tcp_server_posix.cc +3 -3
  238. data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +4 -6
  239. data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +7 -7
  240. data/src/core/lib/iomgr/tcp_server_windows.cc +10 -16
  241. data/src/core/lib/iomgr/tcp_windows.cc +25 -41
  242. data/src/core/lib/iomgr/timer_generic.cc +17 -20
  243. data/src/core/lib/iomgr/timer_heap.cc +1 -1
  244. data/src/core/lib/iomgr/timer_manager.cc +17 -30
  245. data/src/core/lib/iomgr/unix_sockets_posix.cc +1 -1
  246. data/src/core/lib/iomgr/vsock.cc +1 -1
  247. data/src/core/lib/iomgr/wakeup_fd_pipe.cc +3 -3
  248. data/src/core/lib/promise/activity.h +27 -4
  249. data/src/core/lib/promise/cancel_callback.h +24 -0
  250. data/src/core/lib/promise/context.h +11 -0
  251. data/src/core/lib/promise/detail/basic_seq.h +1 -2
  252. data/src/core/lib/promise/detail/join_state.h +354 -398
  253. data/src/core/lib/promise/detail/promise_like.h +13 -6
  254. data/src/core/lib/promise/detail/seq_state.h +1178 -1178
  255. data/src/core/lib/promise/for_each.h +6 -6
  256. data/src/core/lib/promise/interceptor_list.h +6 -7
  257. data/src/core/lib/promise/latch.h +9 -9
  258. data/src/core/lib/promise/map.h +17 -0
  259. data/src/core/lib/promise/observable.h +182 -0
  260. data/src/core/lib/promise/party.cc +7 -8
  261. data/src/core/lib/promise/party.h +10 -8
  262. data/src/core/lib/promise/pipe.h +16 -35
  263. data/src/core/lib/promise/promise.h +1 -0
  264. data/src/core/lib/promise/status_flag.h +2 -0
  265. data/src/core/lib/resource_quota/arena.cc +56 -79
  266. data/src/core/lib/resource_quota/arena.h +118 -209
  267. data/src/core/lib/resource_quota/memory_quota.cc +12 -13
  268. data/src/core/lib/resource_quota/memory_quota.h +2 -3
  269. data/src/core/lib/resource_quota/periodic_update.cc +1 -1
  270. data/src/core/lib/resource_quota/resource_quota.h +1 -1
  271. data/src/core/lib/security/authorization/authorization_policy_provider.h +1 -1
  272. data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +1 -1
  273. data/src/core/lib/security/authorization/evaluate_args.cc +6 -8
  274. data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +5 -6
  275. data/src/core/lib/security/authorization/grpc_server_authz_filter.h +1 -0
  276. data/src/core/lib/security/authorization/matchers.cc +3 -3
  277. data/src/core/lib/security/certificate_provider/certificate_provider_factory.h +2 -2
  278. data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +2 -3
  279. data/src/core/lib/security/context/security_context.cc +12 -13
  280. data/src/core/lib/security/context/security_context.h +31 -8
  281. data/src/core/lib/security/credentials/alts/alts_credentials.h +1 -1
  282. data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +3 -1
  283. data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +2 -3
  284. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +5 -5
  285. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +3 -3
  286. data/src/core/lib/security/credentials/call_creds_util.cc +2 -1
  287. data/src/core/lib/security/credentials/channel_creds_registry.h +2 -2
  288. data/src/core/lib/security/credentials/channel_creds_registry_init.cc +5 -3
  289. data/src/core/lib/security/credentials/composite/composite_credentials.h +1 -1
  290. data/src/core/lib/security/credentials/credentials.cc +6 -6
  291. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +4 -4
  292. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +2 -2
  293. data/src/core/lib/security/credentials/external/external_account_credentials.cc +9 -11
  294. data/src/core/lib/security/credentials/external/external_account_credentials.h +3 -3
  295. data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +2 -2
  296. data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +4 -4
  297. data/src/core/lib/security/credentials/external/url_external_account_credentials.h +1 -1
  298. data/src/core/lib/security/credentials/fake/fake_credentials.h +1 -1
  299. data/src/core/lib/security/credentials/google_default/credentials_generic.cc +3 -3
  300. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +11 -11
  301. data/src/core/lib/security/credentials/google_default/google_default_credentials.h +1 -1
  302. data/src/core/lib/security/credentials/iam/iam_credentials.h +1 -1
  303. data/src/core/lib/security/credentials/jwt/json_token.cc +14 -15
  304. data/src/core/lib/security/credentials/jwt/json_token.h +1 -1
  305. data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +6 -5
  306. data/src/core/lib/security/credentials/jwt/jwt_credentials.h +1 -1
  307. data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +50 -54
  308. data/src/core/lib/security/credentials/jwt/jwt_verifier.h +1 -1
  309. data/src/core/lib/security/credentials/local/local_credentials.h +1 -1
  310. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +12 -11
  311. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +4 -4
  312. data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +6 -7
  313. data/src/core/lib/security/credentials/plugin/plugin_credentials.h +1 -3
  314. data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +15 -21
  315. data/src/core/lib/security/credentials/ssl/ssl_credentials.h +1 -1
  316. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +21 -30
  317. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +1 -1
  318. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +1 -1
  319. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +4 -4
  320. data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +3 -5
  321. data/src/core/lib/security/credentials/tls/tls_credentials.cc +14 -16
  322. data/src/core/lib/security/credentials/tls/tls_utils.cc +4 -4
  323. data/src/core/lib/security/credentials/xds/xds_credentials.cc +1 -1
  324. data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +13 -16
  325. data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +15 -12
  326. data/src/core/lib/security/security_connector/load_system_roots_supported.cc +6 -6
  327. data/src/core/lib/security/security_connector/load_system_roots_windows.cc +1 -1
  328. data/src/core/lib/security/security_connector/local/local_security_connector.cc +8 -12
  329. data/src/core/lib/security/security_connector/security_connector.cc +1 -4
  330. data/src/core/lib/security/security_connector/security_connector.h +1 -3
  331. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +17 -19
  332. data/src/core/lib/security/security_connector/ssl_utils.cc +19 -21
  333. data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +29 -40
  334. data/src/core/lib/security/transport/auth_filters.h +1 -0
  335. data/src/core/lib/security/transport/client_auth_filter.cc +7 -13
  336. data/src/core/lib/security/transport/server_auth_filter.cc +3 -8
  337. data/src/core/lib/security/util/json_util.h +1 -1
  338. data/src/core/lib/slice/slice.h +1 -1
  339. data/src/core/lib/slice/slice_refcount.h +2 -4
  340. data/src/core/lib/slice/slice_string_helpers.cc +1 -1
  341. data/src/core/lib/surface/api_trace.h +1 -3
  342. data/src/core/lib/surface/call.cc +64 -3739
  343. data/src/core/lib/surface/call.h +41 -143
  344. data/src/core/lib/surface/call_log_batch.cc +1 -1
  345. data/src/core/lib/surface/call_utils.cc +276 -0
  346. data/src/core/lib/surface/call_utils.h +449 -0
  347. data/src/core/lib/surface/channel.cc +8 -3
  348. data/src/core/lib/surface/channel.h +10 -7
  349. data/src/core/lib/surface/channel_create.cc +14 -6
  350. data/src/core/lib/surface/channel_create.h +3 -2
  351. data/src/core/lib/surface/channel_init.cc +21 -77
  352. data/src/core/lib/surface/channel_init.h +19 -97
  353. data/src/core/lib/surface/client_call.cc +419 -0
  354. data/src/core/lib/surface/client_call.h +180 -0
  355. data/src/core/lib/surface/completion_queue.cc +28 -33
  356. data/src/core/lib/surface/completion_queue.h +0 -8
  357. data/src/core/lib/surface/filter_stack_call.cc +1157 -0
  358. data/src/core/lib/surface/filter_stack_call.h +369 -0
  359. data/src/core/lib/surface/init.cc +7 -6
  360. data/src/core/lib/surface/lame_client.cc +1 -1
  361. data/src/core/lib/surface/legacy_channel.cc +43 -30
  362. data/src/core/lib/surface/legacy_channel.h +9 -18
  363. data/src/core/lib/surface/server_call.cc +222 -0
  364. data/src/core/lib/surface/server_call.h +167 -0
  365. data/src/core/lib/surface/version.cc +2 -2
  366. data/src/core/lib/transport/bdp_estimator.cc +3 -5
  367. data/src/core/lib/transport/bdp_estimator.h +2 -4
  368. data/src/core/lib/transport/call_arena_allocator.h +9 -7
  369. data/src/core/lib/transport/call_destination.h +76 -0
  370. data/src/core/lib/transport/call_filters.cc +28 -10
  371. data/src/core/lib/transport/call_filters.h +128 -22
  372. data/src/core/lib/transport/call_spine.cc +5 -6
  373. data/src/core/lib/transport/call_spine.h +159 -334
  374. data/src/core/lib/transport/connectivity_state.cc +8 -10
  375. data/src/core/lib/transport/connectivity_state.h +0 -2
  376. data/src/core/lib/transport/interception_chain.cc +155 -0
  377. data/src/core/lib/transport/interception_chain.h +236 -0
  378. data/src/core/lib/transport/metadata_batch.h +10 -1
  379. data/src/core/lib/transport/metadata_info.h +1 -1
  380. data/src/core/lib/transport/transport.cc +3 -6
  381. data/src/core/lib/transport/transport.h +43 -40
  382. data/src/core/load_balancing/child_policy_handler.cc +8 -8
  383. data/src/core/load_balancing/endpoint_list.cc +5 -5
  384. data/src/core/load_balancing/endpoint_list.h +1 -1
  385. data/src/core/load_balancing/grpclb/client_load_reporting_filter.cc +1 -0
  386. data/src/core/load_balancing/grpclb/client_load_reporting_filter.h +1 -0
  387. data/src/core/load_balancing/grpclb/grpclb.cc +25 -29
  388. data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc +1 -1
  389. data/src/core/load_balancing/grpclb/load_balancer_api.cc +3 -4
  390. data/src/core/load_balancing/health_check_client.cc +10 -13
  391. data/src/core/load_balancing/lb_policy.cc +5 -8
  392. data/src/core/load_balancing/lb_policy.h +19 -3
  393. data/src/core/load_balancing/lb_policy_factory.h +1 -1
  394. data/src/core/load_balancing/lb_policy_registry.cc +2 -3
  395. data/src/core/load_balancing/lb_policy_registry.h +1 -1
  396. data/src/core/load_balancing/oob_backend_metric.cc +2 -4
  397. data/src/core/load_balancing/outlier_detection/outlier_detection.cc +33 -35
  398. data/src/core/load_balancing/outlier_detection/outlier_detection.h +3 -3
  399. data/src/core/load_balancing/pick_first/pick_first.cc +65 -65
  400. data/src/core/load_balancing/priority/priority.cc +26 -28
  401. data/src/core/load_balancing/ring_hash/ring_hash.cc +11 -13
  402. data/src/core/load_balancing/ring_hash/ring_hash.h +3 -3
  403. data/src/core/load_balancing/rls/rls.cc +82 -82
  404. data/src/core/load_balancing/round_robin/round_robin.cc +17 -20
  405. data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +54 -43
  406. data/src/core/load_balancing/weighted_target/weighted_target.cc +21 -24
  407. data/src/core/load_balancing/xds/cds.cc +14 -16
  408. data/src/core/load_balancing/xds/xds_cluster_impl.cc +16 -18
  409. data/src/core/load_balancing/xds/xds_cluster_manager.cc +15 -17
  410. data/src/core/load_balancing/xds/xds_override_host.cc +40 -41
  411. data/src/core/load_balancing/xds/xds_override_host.h +3 -3
  412. data/src/core/load_balancing/xds/xds_wrr_locality.cc +10 -12
  413. data/src/core/plugin_registry/grpc_plugin_registry.cc +5 -1
  414. data/src/core/resolver/binder/binder_resolver.cc +3 -2
  415. data/src/core/resolver/dns/c_ares/dns_resolver_ares.cc +3 -2
  416. data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +1 -1
  417. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc +7 -14
  418. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.h +1 -5
  419. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +1 -1
  420. data/src/core/resolver/dns/dns_resolver_plugin.cc +6 -5
  421. data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc +4 -9
  422. data/src/core/resolver/dns/event_engine/service_config_helper.cc +5 -5
  423. data/src/core/resolver/dns/native/dns_resolver.cc +8 -9
  424. data/src/core/resolver/endpoint_addresses.cc +1 -1
  425. data/src/core/resolver/fake/fake_resolver.cc +1 -1
  426. data/src/core/resolver/fake/fake_resolver.h +1 -1
  427. data/src/core/resolver/google_c2p/google_c2p_resolver.cc +13 -14
  428. data/src/core/resolver/polling_resolver.cc +30 -35
  429. data/src/core/resolver/resolver.cc +2 -6
  430. data/src/core/resolver/resolver.h +0 -2
  431. data/src/core/resolver/resolver_registry.cc +6 -8
  432. data/src/core/resolver/sockaddr/sockaddr_resolver.cc +3 -3
  433. data/src/core/resolver/xds/xds_dependency_manager.cc +22 -23
  434. data/src/core/resolver/xds/xds_resolver.cc +13 -15
  435. data/src/core/server/server.cc +269 -389
  436. data/src/core/server/server.h +37 -19
  437. data/src/core/server/server_call_tracer_filter.cc +7 -14
  438. data/src/core/server/server_config_selector.h +1 -1
  439. data/src/core/server/server_config_selector_filter.cc +3 -3
  440. data/src/core/server/server_interface.h +2 -0
  441. data/src/core/server/xds_channel_stack_modifier.cc +1 -1
  442. data/src/core/server/xds_channel_stack_modifier.h +1 -1
  443. data/src/core/server/xds_server_config_fetcher.cc +1 -4
  444. data/src/core/service_config/service_config.h +1 -1
  445. data/src/core/service_config/service_config_call_data.h +13 -11
  446. data/src/core/service_config/service_config_channel_arg_filter.cc +6 -4
  447. data/src/core/service_config/service_config_impl.cc +5 -5
  448. data/src/core/service_config/service_config_impl.h +1 -1
  449. data/src/core/service_config/service_config_parser.cc +3 -6
  450. data/src/core/service_config/service_config_parser.h +1 -1
  451. data/src/core/{lib/channel → telemetry}/call_tracer.cc +20 -30
  452. data/src/core/{lib/channel → telemetry}/call_tracer.h +32 -9
  453. data/src/core/{lib/debug → telemetry}/histogram_view.cc +1 -1
  454. data/src/core/{lib/debug → telemetry}/histogram_view.h +3 -3
  455. data/src/core/telemetry/metrics.cc +178 -0
  456. data/src/core/telemetry/metrics.h +562 -0
  457. data/src/core/{lib/debug → telemetry}/stats.cc +1 -1
  458. data/src/core/{lib/debug → telemetry}/stats.h +5 -5
  459. data/src/core/{lib/debug → telemetry}/stats_data.cc +1 -1
  460. data/src/core/{lib/debug → telemetry}/stats_data.h +4 -4
  461. data/src/core/{lib/channel → telemetry}/tcp_tracer.h +3 -3
  462. data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +12 -13
  463. data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +25 -27
  464. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +32 -33
  465. data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +2 -1
  466. data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +2 -1
  467. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +5 -4
  468. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +5 -3
  469. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +1 -1
  470. data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +4 -3
  471. data/src/core/tsi/fake_transport_security.cc +14 -17
  472. data/src/core/tsi/local_transport_security.cc +6 -5
  473. data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +4 -4
  474. data/src/core/tsi/ssl_transport_security.cc +76 -81
  475. data/src/core/tsi/ssl_transport_security_utils.cc +74 -18
  476. data/src/core/tsi/ssl_transport_security_utils.h +11 -0
  477. data/src/core/tsi/transport_security.cc +0 -4
  478. data/src/core/tsi/transport_security.h +0 -2
  479. data/src/core/tsi/transport_security_interface.h +0 -4
  480. data/src/core/{lib/gpr → util}/alloc.h +3 -3
  481. data/src/core/{lib/gpr → util}/android/log.cc +0 -19
  482. data/src/core/{lib/gpr → util}/atm.cc +1 -1
  483. data/src/core/{ext/gcp/metadata_query.cc → util/gcp_metadata_query.cc} +25 -26
  484. data/src/core/{ext/gcp/metadata_query.h → util/gcp_metadata_query.h} +11 -11
  485. data/src/core/{lib/http → util/http_client}/format_request.cc +4 -3
  486. data/src/core/{lib/http → util/http_client}/format_request.h +6 -5
  487. data/src/core/{lib/http → util/http_client}/httpcli.cc +9 -10
  488. data/src/core/{lib/http → util/http_client}/httpcli.h +6 -5
  489. data/src/core/{lib/http → util/http_client}/httpcli_security_connector.cc +9 -9
  490. data/src/core/{lib/http → util/http_client}/httpcli_ssl_credentials.h +5 -4
  491. data/src/core/{lib/http → util/http_client}/parser.cc +4 -5
  492. data/src/core/{lib/http → util/http_client}/parser.h +5 -6
  493. data/src/core/{lib → util}/json/json.h +5 -4
  494. data/src/core/{lib → util}/json/json_args.h +5 -5
  495. data/src/core/{lib → util}/json/json_channel_args.h +6 -6
  496. data/src/core/{lib → util}/json/json_object_loader.cc +3 -2
  497. data/src/core/{lib → util}/json/json_object_loader.h +7 -7
  498. data/src/core/{lib → util}/json/json_reader.cc +3 -2
  499. data/src/core/{lib → util}/json/json_reader.h +6 -6
  500. data/src/core/{lib → util}/json/json_util.cc +4 -4
  501. data/src/core/{lib → util}/json/json_util.h +6 -6
  502. data/src/core/{lib → util}/json/json_writer.cc +3 -3
  503. data/src/core/{lib → util}/json/json_writer.h +6 -6
  504. data/src/core/{lib/gpr → util}/linux/log.cc +0 -45
  505. data/src/core/{lib/gpr → util}/log.cc +45 -12
  506. data/src/core/{lib/gpr → util}/msys/tmpfile.cc +2 -2
  507. data/src/core/{lib/gpr → util}/posix/cpu.cc +1 -1
  508. data/src/core/{lib/gpr → util}/posix/log.cc +0 -42
  509. data/src/core/{lib/gpr → util}/posix/time.cc +1 -1
  510. data/src/core/{lib/gpr → util}/posix/tmpfile.cc +2 -2
  511. data/src/core/{lib/gpr → util}/spinlock.h +3 -3
  512. data/src/core/{lib/gpr → util}/string.cc +2 -2
  513. data/src/core/{lib/gpr → util}/string.h +3 -3
  514. data/src/core/{lib/gpr → util}/time_precise.cc +1 -1
  515. data/src/core/{lib/gpr → util}/time_precise.h +3 -3
  516. data/src/core/{lib/gpr → util}/tmpfile.h +3 -3
  517. data/src/core/{lib/gpr → util}/useful.h +3 -3
  518. data/src/core/{lib/gpr → util}/windows/log.cc +1 -44
  519. data/src/core/{lib/gpr → util}/windows/string.cc +1 -1
  520. data/src/core/{lib/gpr → util}/windows/string_util.cc +1 -1
  521. data/src/core/{lib/gpr → util}/windows/time.cc +1 -1
  522. data/src/core/{lib/gpr → util}/windows/tmpfile.cc +1 -1
  523. data/src/core/xds/grpc/certificate_provider_store.cc +3 -3
  524. data/src/core/xds/grpc/certificate_provider_store.h +4 -4
  525. data/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc +3 -3
  526. data/src/core/xds/grpc/file_watcher_certificate_provider_factory.h +3 -3
  527. data/src/core/xds/grpc/xds_audit_logger_registry.h +1 -1
  528. data/src/core/xds/grpc/xds_bootstrap_grpc.cc +5 -5
  529. data/src/core/xds/grpc/xds_bootstrap_grpc.h +3 -3
  530. data/src/core/xds/grpc/xds_certificate_provider.h +1 -1
  531. data/src/core/xds/grpc/xds_client_grpc.cc +27 -23
  532. data/src/core/xds/grpc/xds_client_grpc.h +2 -2
  533. data/src/core/xds/grpc/xds_cluster.cc +4 -5
  534. data/src/core/xds/grpc/xds_cluster.h +1 -1
  535. data/src/core/xds/grpc/xds_cluster_specifier_plugin.cc +2 -2
  536. data/src/core/xds/grpc/xds_cluster_specifier_plugin.h +1 -1
  537. data/src/core/xds/grpc/xds_common_types.cc +1 -1
  538. data/src/core/xds/grpc/xds_common_types.h +1 -1
  539. data/src/core/xds/grpc/xds_endpoint.cc +4 -5
  540. data/src/core/xds/grpc/xds_http_fault_filter.cc +2 -2
  541. data/src/core/xds/grpc/xds_http_filters.h +2 -2
  542. data/src/core/xds/grpc/xds_http_rbac_filter.cc +3 -3
  543. data/src/core/xds/grpc/xds_http_stateful_session_filter.cc +2 -2
  544. data/src/core/xds/grpc/xds_lb_policy_registry.h +1 -1
  545. data/src/core/xds/grpc/xds_listener.cc +4 -6
  546. data/src/core/xds/grpc/xds_route_config.cc +7 -8
  547. data/src/core/xds/grpc/xds_transport_grpc.cc +2 -2
  548. data/src/core/xds/grpc/xds_transport_grpc.h +1 -1
  549. data/src/core/xds/xds_client/xds_api.cc +5 -9
  550. data/src/core/xds/xds_client/xds_bootstrap.cc +1 -1
  551. data/src/core/xds/xds_client/xds_bootstrap.h +1 -1
  552. data/src/core/xds/xds_client/xds_client.cc +39 -45
  553. data/src/core/xds/xds_client/xds_client.h +0 -3
  554. data/src/core/xds/xds_client/xds_client_stats.cc +20 -18
  555. data/src/core/xds/xds_client/xds_client_stats.h +2 -2
  556. data/src/ruby/ext/grpc/rb_call.c +8 -1
  557. data/src/ruby/ext/grpc/rb_completion_queue.c +15 -32
  558. data/src/ruby/ext/grpc/rb_completion_queue.h +7 -1
  559. data/src/ruby/ext/grpc/rb_server.c +39 -22
  560. data/src/ruby/lib/grpc/version.rb +1 -1
  561. data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +4 -0
  562. data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +12 -12
  563. data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +66 -41
  564. data/third_party/boringssl-with-bazel/src/crypto/dilithium/dilithium.c +1497 -0
  565. data/third_party/boringssl-with-bazel/src/crypto/dilithium/internal.h +58 -0
  566. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +10 -3
  567. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +0 -2
  568. data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +2 -0
  569. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +5 -0
  570. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +45 -1
  571. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +33 -23
  572. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/fips_shared_support.c +3 -6
  573. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +9 -4
  574. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +6 -0
  575. data/third_party/boringssl-with-bazel/src/crypto/internal.h +7 -0
  576. data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +0 -4
  577. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_utl.c +49 -16
  578. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +0 -10
  579. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +7 -17
  580. data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +8 -6
  581. data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +3 -0
  582. data/third_party/boringssl-with-bazel/src/include/openssl/experimental/dilithium.h +125 -0
  583. data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +0 -23
  584. data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +1 -1
  585. data/third_party/boringssl-with-bazel/src/ssl/internal.h +4 -10
  586. metadata +95 -91
  587. data/src/core/ext/transport/chttp2/transport/http_trace.cc +0 -19
  588. data/src/core/ext/transport/chttp2/transport/http_trace.h +0 -24
  589. data/src/core/ext/transport/inproc/inproc_plugin.cc +0 -23
  590. data/src/core/handshaker/security/tsi_error.cc +0 -31
  591. data/src/core/handshaker/security/tsi_error.h +0 -30
  592. data/src/core/lib/channel/channel_stack_trace.cc +0 -19
  593. data/src/core/lib/channel/channel_stack_trace.h +0 -24
  594. data/src/core/lib/channel/context.h +0 -105
  595. data/src/core/lib/channel/metrics.cc +0 -334
  596. data/src/core/lib/channel/metrics.h +0 -365
  597. data/src/core/lib/event_engine/trace.cc +0 -25
  598. data/src/core/lib/iomgr/ev_windows.cc +0 -30
  599. data/src/core/lib/promise/trace.cc +0 -20
  600. data/src/core/lib/promise/trace.h +0 -24
  601. data/src/core/lib/resource_quota/trace.cc +0 -19
  602. data/src/core/lib/resource_quota/trace.h +0 -24
  603. data/src/core/lib/slice/slice_refcount.cc +0 -20
  604. data/src/core/lib/surface/api_trace.cc +0 -25
  605. data/src/core/lib/surface/call_trace.h +0 -24
  606. data/src/core/lib/surface/wait_for_cq_end_op.cc +0 -75
  607. data/src/core/lib/surface/wait_for_cq_end_op.h +0 -72
  608. data/src/core/lib/transport/batch_builder.cc +0 -172
  609. data/src/core/lib/transport/batch_builder.h +0 -474
  610. data/src/core/resolver/xds/xds_resolver_trace.cc +0 -25
  611. data/src/core/resolver/xds/xds_resolver_trace.h +0 -30
  612. data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -122
  613. /data/src/core/{lib/gpr → util}/alloc.cc +0 -0
  614. /data/src/core/{lib/gpr → util}/iphone/cpu.cc +0 -0
  615. /data/src/core/{lib/gpr → util}/linux/cpu.cc +0 -0
  616. /data/src/core/{lib/gpr → util}/posix/string.cc +0 -0
  617. /data/src/core/{lib/gpr → util}/posix/sync.cc +0 -0
  618. /data/src/core/{lib/gpr → util}/sync.cc +0 -0
  619. /data/src/core/{lib/gpr → util}/sync_abseil.cc +0 -0
  620. /data/src/core/{lib/gpr → util}/time.cc +0 -0
  621. /data/src/core/{lib/gpr → util}/windows/cpu.cc +0 -0
  622. /data/src/core/{lib/gpr → util}/windows/sync.cc +0 -0
@@ -25,8 +25,10 @@
25
25
 
26
26
  #include <algorithm>
27
27
  #include <atomic>
28
+ #include <cstdint>
28
29
  #include <memory>
29
30
  #include <new>
31
+ #include <queue>
30
32
  #include <string>
31
33
  #include <type_traits>
32
34
  #include <utility>
@@ -34,6 +36,7 @@
34
36
 
35
37
  #include "absl/base/thread_annotations.h"
36
38
  #include "absl/log/check.h"
39
+ #include "absl/log/log.h"
37
40
  #include "absl/status/status.h"
38
41
  #include "absl/strings/str_cat.h"
39
42
  #include "absl/strings/str_format.h"
@@ -57,21 +60,15 @@
57
60
 
58
61
  #include "src/core/channelz/channelz.h"
59
62
  #include "src/core/lib/channel/call_finalization.h"
60
- #include "src/core/lib/channel/call_tracer.h"
61
63
  #include "src/core/lib/channel/channel_stack.h"
62
- #include "src/core/lib/channel/context.h"
63
64
  #include "src/core/lib/channel/status_util.h"
64
65
  #include "src/core/lib/compression/compression_internal.h"
65
- #include "src/core/lib/debug/stats.h"
66
- #include "src/core/lib/debug/stats_data.h"
67
66
  #include "src/core/lib/experiments/experiments.h"
68
- #include "src/core/lib/gpr/alloc.h"
69
- #include "src/core/lib/gpr/time_precise.h"
70
- #include "src/core/lib/gpr/useful.h"
71
67
  #include "src/core/lib/gprpp/bitset.h"
72
68
  #include "src/core/lib/gprpp/cpp_impl_of.h"
73
69
  #include "src/core/lib/gprpp/crash.h"
74
70
  #include "src/core/lib/gprpp/debug_location.h"
71
+ #include "src/core/lib/gprpp/match.h"
75
72
  #include "src/core/lib/gprpp/ref_counted.h"
76
73
  #include "src/core/lib/gprpp/ref_counted_ptr.h"
77
74
  #include "src/core/lib/gprpp/status_helper.h"
@@ -82,10 +79,10 @@
82
79
  #include "src/core/lib/promise/activity.h"
83
80
  #include "src/core/lib/promise/all_ok.h"
84
81
  #include "src/core/lib/promise/arena_promise.h"
82
+ #include "src/core/lib/promise/cancel_callback.h"
85
83
  #include "src/core/lib/promise/context.h"
86
84
  #include "src/core/lib/promise/latch.h"
87
85
  #include "src/core/lib/promise/map.h"
88
- #include "src/core/lib/promise/party.h"
89
86
  #include "src/core/lib/promise/pipe.h"
90
87
  #include "src/core/lib/promise/poll.h"
91
88
  #include "src/core/lib/promise/race.h"
@@ -100,17 +97,17 @@
100
97
  #include "src/core/lib/surface/channel.h"
101
98
  #include "src/core/lib/surface/completion_queue.h"
102
99
  #include "src/core/lib/surface/validate_metadata.h"
103
- #include "src/core/lib/surface/wait_for_cq_end_op.h"
104
- #include "src/core/lib/transport/batch_builder.h"
105
100
  #include "src/core/lib/transport/error_utils.h"
101
+ #include "src/core/lib/transport/metadata.h"
106
102
  #include "src/core/lib/transport/metadata_batch.h"
107
103
  #include "src/core/lib/transport/transport.h"
108
104
  #include "src/core/server/server_interface.h"
109
-
110
- grpc_core::TraceFlag grpc_call_error_trace(false, "call_error");
111
- grpc_core::TraceFlag grpc_compression_trace(false, "compression");
112
- grpc_core::TraceFlag grpc_call_trace(false, "call");
113
- grpc_core::DebugOnlyTraceFlag grpc_call_refcount_trace(false, "call_refcount");
105
+ #include "src/core/telemetry/call_tracer.h"
106
+ #include "src/core/telemetry/stats.h"
107
+ #include "src/core/telemetry/stats_data.h"
108
+ #include "src/core/util/alloc.h"
109
+ #include "src/core/util/time_precise.h"
110
+ #include "src/core/util/useful.h"
114
111
 
115
112
  namespace grpc_core {
116
113
 
@@ -121,10 +118,19 @@ using GrpcClosure = Closure;
121
118
  ///////////////////////////////////////////////////////////////////////////////
122
119
  // Call
123
120
 
121
+ Call::Call(bool is_client, Timestamp send_deadline, RefCountedPtr<Arena> arena,
122
+ grpc_event_engine::experimental::EventEngine* event_engine)
123
+ : arena_(std::move(arena)),
124
+ send_deadline_(send_deadline),
125
+ is_client_(is_client),
126
+ event_engine_(event_engine) {
127
+ arena_->SetContext<Call>(this);
128
+ }
129
+
124
130
  Call::ParentCall* Call::GetOrCreateParentCall() {
125
131
  ParentCall* p = parent_call_.load(std::memory_order_acquire);
126
132
  if (p == nullptr) {
127
- p = arena_->New<ParentCall>();
133
+ p = arena()->New<ParentCall>();
128
134
  ParentCall* expected = nullptr;
129
135
  if (!parent_call_.compare_exchange_strong(expected, p,
130
136
  std::memory_order_release,
@@ -160,8 +166,8 @@ absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) {
160
166
  "Census tracing propagation requested without Census context "
161
167
  "propagation");
162
168
  }
163
- ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING),
164
- nullptr);
169
+ arena()->SetContext<census_context>(
170
+ parent->arena()->GetContext<census_context>());
165
171
  } else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) {
166
172
  return absl::UnknownError(
167
173
  "Census context propagation requested without Census tracing "
@@ -243,28 +249,6 @@ void Call::PropagateCancellationToChildren() {
243
249
  }
244
250
  }
245
251
 
246
- char* Call::GetPeer() {
247
- Slice peer_slice = GetPeerString();
248
- if (!peer_slice.empty()) {
249
- absl::string_view peer_string_view = peer_slice.as_string_view();
250
- char* peer_string =
251
- static_cast<char*>(gpr_malloc(peer_string_view.size() + 1));
252
- memcpy(peer_string, peer_string_view.data(), peer_string_view.size());
253
- peer_string[peer_string_view.size()] = '\0';
254
- return peer_string;
255
- }
256
- char* peer_string = grpc_channel_get_target(channel_->c_ptr());
257
- if (peer_string != nullptr) return peer_string;
258
- return gpr_strdup("unknown");
259
- }
260
-
261
- void Call::DeleteThis() {
262
- RefCountedPtr<Channel> channel = std::move(channel_);
263
- Arena* arena = arena_;
264
- this->~Call();
265
- channel->DestroyArena(arena);
266
- }
267
-
268
252
  void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
269
253
  grpc_metadata_batch& md) {
270
254
  // TODO(juanlishen): If the user has already specified a compression
@@ -279,7 +263,7 @@ void Call::PrepareOutgoingInitialMetadata(const grpc_op& op,
279
263
  op.data.send_initial_metadata.maybe_compression_level.level;
280
264
  level_set = true;
281
265
  } else {
282
- const grpc_compression_options copts = channel()->compression_options();
266
+ const grpc_compression_options copts = compression_options();
283
267
  if (copts.default_level.is_set) {
284
268
  level_set = true;
285
269
  effective_compression_level = copts.default_level.level;
@@ -305,26 +289,25 @@ void Call::ProcessIncomingInitialMetadata(grpc_metadata_batch& md) {
305
289
  Slice* peer_string = md.get_pointer(PeerString());
306
290
  if (peer_string != nullptr) SetPeerString(peer_string->Ref());
307
291
 
308
- incoming_compression_algorithm_ =
309
- md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE);
292
+ SetIncomingCompressionAlgorithm(
293
+ md.Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE));
310
294
  encodings_accepted_by_peer_ =
311
295
  md.Take(GrpcAcceptEncodingMetadata())
312
296
  .value_or(CompressionAlgorithmSet{GRPC_COMPRESS_NONE});
313
297
 
314
- const grpc_compression_options compression_options =
315
- channel_->compression_options();
298
+ const grpc_compression_options copts = compression_options();
316
299
  const grpc_compression_algorithm compression_algorithm =
317
- incoming_compression_algorithm_;
318
- if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32(
319
- compression_options.enabled_algorithms_bitset)
320
- .IsSet(compression_algorithm))) {
300
+ incoming_compression_algorithm();
301
+ if (GPR_UNLIKELY(
302
+ !CompressionAlgorithmSet::FromUint32(copts.enabled_algorithms_bitset)
303
+ .IsSet(compression_algorithm))) {
321
304
  // check if algorithm is supported by current channel config
322
305
  HandleCompressionAlgorithmDisabled(compression_algorithm);
323
306
  }
324
307
  // GRPC_COMPRESS_NONE is always set.
325
308
  DCHECK(encodings_accepted_by_peer_.IsSet(GRPC_COMPRESS_NONE));
326
309
  if (GPR_UNLIKELY(!encodings_accepted_by_peer_.IsSet(compression_algorithm))) {
327
- if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
310
+ if (GRPC_TRACE_FLAG_ENABLED(compression)) {
328
311
  HandleCompressionAlgorithmNotAccepted(compression_algorithm);
329
312
  }
330
313
  }
@@ -347,7 +330,7 @@ void Call::HandleCompressionAlgorithmDisabled(
347
330
  grpc_compression_algorithm_name(compression_algorithm, &algo_name);
348
331
  std::string error_msg =
349
332
  absl::StrFormat("Compression algorithm '%s' is disabled.", algo_name);
350
- gpr_log(GPR_ERROR, "%s", error_msg.c_str());
333
+ LOG(ERROR) << error_msg;
351
334
  CancelWithError(grpc_error_set_int(absl::UnimplementedError(error_msg),
352
335
  StatusIntProperty::kRpcStatus,
353
336
  GRPC_STATUS_UNIMPLEMENTED));
@@ -355,7 +338,7 @@ void Call::HandleCompressionAlgorithmDisabled(
355
338
 
356
339
  void Call::UpdateDeadline(Timestamp deadline) {
357
340
  ReleasableMutexLock lock(&deadline_mu_);
358
- if (grpc_call_trace.enabled()) {
341
+ if (GRPC_TRACE_FLAG_ENABLED(call)) {
359
342
  gpr_log(GPR_DEBUG, "[call %p] UpdateDeadline from=%s to=%s", this,
360
343
  deadline_.ToString().c_str(), deadline.ToString().c_str());
361
344
  }
@@ -367,22 +350,20 @@ void Call::UpdateDeadline(Timestamp deadline) {
367
350
  StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
368
351
  return;
369
352
  }
370
- auto* const event_engine = channel()->event_engine();
371
353
  if (deadline_ != Timestamp::InfFuture()) {
372
- if (!event_engine->Cancel(deadline_task_)) return;
354
+ if (!event_engine_->Cancel(deadline_task_)) return;
373
355
  } else {
374
356
  InternalRef("deadline");
375
357
  }
376
358
  deadline_ = deadline;
377
- deadline_task_ = event_engine->RunAfter(deadline - Timestamp::Now(), this);
359
+ deadline_task_ = event_engine_->RunAfter(deadline - Timestamp::Now(), this);
378
360
  }
379
361
 
380
362
  void Call::ResetDeadline() {
381
363
  {
382
364
  MutexLock lock(&deadline_mu_);
383
365
  if (deadline_ == Timestamp::InfFuture()) return;
384
- auto* const event_engine = channel()->event_engine();
385
- if (!event_engine->Cancel(deadline_task_)) return;
366
+ if (!event_engine_->Cancel(deadline_task_)) return;
386
367
  deadline_ = Timestamp::InfFuture();
387
368
  }
388
369
  InternalUnref("deadline[reset]");
@@ -391,3701 +372,41 @@ void Call::ResetDeadline() {
391
372
  void Call::Run() {
392
373
  ApplicationCallbackExecCtx callback_exec_ctx;
393
374
  ExecCtx exec_ctx;
375
+ GRPC_TRACE_LOG(call, INFO)
376
+ << "call deadline expired "
377
+ << GRPC_DUMP_ARGS(Timestamp::Now(), send_deadline_);
394
378
  CancelWithError(grpc_error_set_int(
395
379
  absl::DeadlineExceededError("Deadline Exceeded"),
396
380
  StatusIntProperty::kRpcStatus, GRPC_STATUS_DEADLINE_EXCEEDED));
397
381
  InternalUnref("deadline[run]");
398
382
  }
399
383
 
400
- ///////////////////////////////////////////////////////////////////////////////
401
- // FilterStackCall
402
- // To be removed once promise conversion is complete
403
-
404
- class FilterStackCall final : public Call {
405
- public:
406
- ~FilterStackCall() override {
407
- for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
408
- if (context_[i].destroy) {
409
- context_[i].destroy(context_[i].value);
410
- }
411
- }
412
- gpr_free(static_cast<void*>(const_cast<char*>(final_info_.error_string)));
413
- }
414
-
415
- bool Completed() override {
416
- return gpr_atm_acq_load(&received_final_op_atm_) != 0;
417
- }
418
-
419
- // TODO(ctiller): return absl::StatusOr<SomeSmartPointer<Call>>?
420
- static grpc_error_handle Create(grpc_call_create_args* args,
421
- grpc_call** out_call);
422
-
423
- static Call* FromTopElem(grpc_call_element* elem) {
424
- return FromCallStack(grpc_call_stack_from_top_element(elem));
425
- }
426
-
427
- grpc_call_stack* call_stack() override {
428
- return reinterpret_cast<grpc_call_stack*>(
429
- reinterpret_cast<char*>(this) +
430
- GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(*this)));
431
- }
432
-
433
- grpc_event_engine::experimental::EventEngine* event_engine() const override {
434
- return channel()->event_engine();
435
- }
436
-
437
- grpc_call_element* call_elem(size_t idx) {
438
- return grpc_call_stack_element(call_stack(), idx);
439
- }
440
-
441
- CallCombiner* call_combiner() { return &call_combiner_; }
442
-
443
- void CancelWithError(grpc_error_handle error) override;
444
- void SetCompletionQueue(grpc_completion_queue* cq) override;
445
- grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
446
- bool is_notify_tag_closure) override;
447
- void ExternalRef() override { ext_ref_.Ref(); }
448
- void ExternalUnref() override;
449
- void InternalRef(const char* reason) override {
450
- GRPC_CALL_STACK_REF(call_stack(), reason);
451
- }
452
- void InternalUnref(const char* reason) override {
453
- GRPC_CALL_STACK_UNREF(call_stack(), reason);
454
- }
455
-
456
- void ContextSet(grpc_context_index elem, void* value,
457
- void (*destroy)(void* value)) override;
458
- void* ContextGet(grpc_context_index elem) const override {
459
- return context_[elem].value;
460
- }
461
-
462
- bool is_trailers_only() const override {
463
- bool result = is_trailers_only_;
464
- DCHECK(!result || recv_initial_metadata_.TransportSize() == 0);
465
- return result;
466
- }
467
-
468
- bool failed_before_recv_message() const override {
469
- return call_failed_before_recv_message_;
470
- }
471
-
472
- absl::string_view GetServerAuthority() const override {
473
- const Slice* authority_metadata =
474
- recv_initial_metadata_.get_pointer(HttpAuthorityMetadata());
475
- if (authority_metadata == nullptr) return "";
476
- return authority_metadata->as_string_view();
477
- }
478
-
479
- static size_t InitialSizeEstimate() {
480
- return sizeof(FilterStackCall) +
481
- sizeof(BatchControl) * kMaxConcurrentBatches;
482
- }
483
-
484
- private:
485
- class ScopedContext : public promise_detail::Context<Arena> {
486
- public:
487
- explicit ScopedContext(FilterStackCall* call)
488
- : promise_detail::Context<Arena>(call->arena()) {}
489
- };
490
-
491
- static constexpr gpr_atm kRecvNone = 0;
492
- static constexpr gpr_atm kRecvInitialMetadataFirst = 1;
493
-
494
- enum class PendingOp {
495
- kRecvMessage,
496
- kRecvInitialMetadata,
497
- kRecvTrailingMetadata,
498
- kSends
499
- };
500
- static intptr_t PendingOpMask(PendingOp op) {
501
- return static_cast<intptr_t>(1) << static_cast<intptr_t>(op);
502
- }
503
- static std::string PendingOpString(intptr_t pending_ops) {
504
- std::vector<absl::string_view> pending_op_strings;
505
- if (pending_ops & PendingOpMask(PendingOp::kRecvMessage)) {
506
- pending_op_strings.push_back("kRecvMessage");
507
- }
508
- if (pending_ops & PendingOpMask(PendingOp::kRecvInitialMetadata)) {
509
- pending_op_strings.push_back("kRecvInitialMetadata");
510
- }
511
- if (pending_ops & PendingOpMask(PendingOp::kRecvTrailingMetadata)) {
512
- pending_op_strings.push_back("kRecvTrailingMetadata");
513
- }
514
- if (pending_ops & PendingOpMask(PendingOp::kSends)) {
515
- pending_op_strings.push_back("kSends");
516
- }
517
- return absl::StrCat("{", absl::StrJoin(pending_op_strings, ","), "}");
518
- }
519
- struct BatchControl {
520
- FilterStackCall* call_ = nullptr;
521
- CallTracerAnnotationInterface* call_tracer_ = nullptr;
522
- grpc_transport_stream_op_batch op_;
523
- // Share memory for cq_completion and notify_tag as they are never needed
524
- // simultaneously. Each byte used in this data structure count as six bytes
525
- // per call, so any savings we can make are worthwhile,
526
-
527
- // We use notify_tag to determine whether or not to send notification to the
528
- // completion queue. Once we've made that determination, we can reuse the
529
- // memory for cq_completion.
530
- union {
531
- grpc_cq_completion cq_completion;
532
- struct {
533
- // Any given op indicates completion by either (a) calling a closure or
534
- // (b) sending a notification on the call's completion queue. If
535
- // \a is_closure is true, \a tag indicates a closure to be invoked;
536
- // otherwise, \a tag indicates the tag to be used in the notification to
537
- // be sent to the completion queue.
538
- void* tag;
539
- bool is_closure;
540
- } notify_tag;
541
- } completion_data_;
542
- grpc_closure start_batch_;
543
- grpc_closure finish_batch_;
544
- std::atomic<intptr_t> ops_pending_{0};
545
- AtomicError batch_error_;
546
- void set_pending_ops(uintptr_t ops) {
547
- ops_pending_.store(ops, std::memory_order_release);
548
- }
549
- bool completed_batch_step(PendingOp op) {
550
- auto mask = PendingOpMask(op);
551
- auto r = ops_pending_.fetch_sub(mask, std::memory_order_acq_rel);
552
- if (grpc_call_trace.enabled()) {
553
- gpr_log(GPR_DEBUG, "BATCH:%p COMPLETE:%s REMAINING:%s (tag:%p)", this,
554
- PendingOpString(mask).c_str(),
555
- PendingOpString(r & ~mask).c_str(),
556
- completion_data_.notify_tag.tag);
557
- }
558
- CHECK_NE((r & mask), 0);
559
- return r == mask;
560
- }
561
-
562
- void PostCompletion();
563
- void FinishStep(PendingOp op);
564
- void ProcessDataAfterMetadata();
565
- void ReceivingStreamReady(grpc_error_handle error);
566
- void ReceivingInitialMetadataReady(grpc_error_handle error);
567
- void ReceivingTrailingMetadataReady(grpc_error_handle error);
568
- void FinishBatch(grpc_error_handle error);
569
- };
570
-
571
- FilterStackCall(Arena* arena, const grpc_call_create_args& args)
572
- : Call(arena, args.server_transport_data == nullptr, args.send_deadline,
573
- args.channel->Ref()),
574
- cq_(args.cq),
575
- stream_op_payload_(context_) {
576
- context_[GRPC_CONTEXT_CALL].value = this;
577
- }
578
-
579
- static void ReleaseCall(void* call, grpc_error_handle);
580
- static void DestroyCall(void* call, grpc_error_handle);
581
-
582
- static FilterStackCall* FromCallStack(grpc_call_stack* call_stack) {
583
- return reinterpret_cast<FilterStackCall*>(
584
- reinterpret_cast<char*>(call_stack) -
585
- GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)));
586
- }
587
-
588
- void ExecuteBatch(grpc_transport_stream_op_batch* batch,
589
- grpc_closure* start_batch_closure);
590
- void SetFinalStatus(grpc_error_handle error);
591
- BatchControl* ReuseOrAllocateBatchControl(const grpc_op* ops);
592
- bool PrepareApplicationMetadata(size_t count, grpc_metadata* metadata,
593
- bool is_trailing);
594
- void PublishAppMetadata(grpc_metadata_batch* b, bool is_trailing);
595
- void RecvInitialFilter(grpc_metadata_batch* b);
596
- void RecvTrailingFilter(grpc_metadata_batch* b,
597
- grpc_error_handle batch_error);
598
-
599
- RefCount ext_ref_;
600
- CallCombiner call_combiner_;
601
- grpc_completion_queue* cq_;
602
- grpc_polling_entity pollent_;
603
-
604
- /// has grpc_call_unref been called
605
- bool destroy_called_ = false;
606
- // Trailers-only response status
607
- bool is_trailers_only_ = false;
608
- /// which ops are in-flight
609
- bool sent_initial_metadata_ = false;
610
- bool sending_message_ = false;
611
- bool sent_final_op_ = false;
612
- bool received_initial_metadata_ = false;
613
- bool receiving_message_ = false;
614
- bool requested_final_op_ = false;
615
- gpr_atm received_final_op_atm_ = 0;
616
-
617
- BatchControl* active_batches_[kMaxConcurrentBatches] = {};
618
- grpc_transport_stream_op_batch_payload stream_op_payload_;
619
-
620
- // first idx: is_receiving, second idx: is_trailing
621
- grpc_metadata_batch send_initial_metadata_;
622
- grpc_metadata_batch send_trailing_metadata_;
623
- grpc_metadata_batch recv_initial_metadata_;
624
- grpc_metadata_batch recv_trailing_metadata_;
625
-
626
- // Buffered read metadata waiting to be returned to the application.
627
- // Element 0 is initial metadata, element 1 is trailing metadata.
628
- grpc_metadata_array* buffered_metadata_[2] = {};
629
-
630
- // Call data useful used for reporting. Only valid after the call has
631
- // completed
632
- grpc_call_final_info final_info_;
633
-
634
- // Contexts for various subsystems (security, tracing, ...).
635
- grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
636
-
637
- SliceBuffer send_slice_buffer_;
638
- absl::optional<SliceBuffer> receiving_slice_buffer_;
639
- uint32_t receiving_stream_flags_;
640
-
641
- bool call_failed_before_recv_message_ = false;
642
- grpc_byte_buffer** receiving_buffer_ = nullptr;
643
- grpc_slice receiving_slice_ = grpc_empty_slice();
644
- grpc_closure receiving_stream_ready_;
645
- grpc_closure receiving_initial_metadata_ready_;
646
- grpc_closure receiving_trailing_metadata_ready_;
647
- // Status about operation of call
648
- bool sent_server_trailing_metadata_ = false;
649
- gpr_atm cancelled_with_error_ = 0;
650
-
651
- grpc_closure release_call_;
652
-
653
- union {
654
- struct {
655
- grpc_status_code* status;
656
- grpc_slice* status_details;
657
- const char** error_string;
658
- } client;
659
- struct {
660
- int* cancelled;
661
- // backpointer to owning server if this is a server side call.
662
- ServerInterface* core_server;
663
- } server;
664
- } final_op_;
665
- AtomicError status_error_;
666
-
667
- // recv_state can contain one of the following values:
668
- // RECV_NONE : : no initial metadata and messages received
669
- // RECV_INITIAL_METADATA_FIRST : received initial metadata first
670
- // a batch_control* : received messages first
671
-
672
- // +------1------RECV_NONE------3-----+
673
- // | |
674
- // | |
675
- // v v
676
- // RECV_INITIAL_METADATA_FIRST receiving_stream_ready_bctlp
677
- // | ^ | ^
678
- // | | | |
679
- // +-----2-----+ +-----4-----+
680
-
681
- // For 1, 4: See receiving_initial_metadata_ready() function
682
- // For 2, 3: See receiving_stream_ready() function
683
- gpr_atm recv_state_ = 0;
684
- };
685
-
686
- grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args,
687
- grpc_call** out_call) {
688
- Channel* channel = args->channel.get();
689
-
690
- auto add_init_error = [](grpc_error_handle* composite,
691
- grpc_error_handle new_err) {
692
- if (new_err.ok()) return;
693
- if (composite->ok()) {
694
- *composite = GRPC_ERROR_CREATE("Call creation failed");
695
- }
696
- *composite = grpc_error_add_child(*composite, new_err);
697
- };
698
-
699
- FilterStackCall* call;
700
- grpc_error_handle error;
701
- grpc_channel_stack* channel_stack = channel->channel_stack();
702
- size_t call_alloc_size =
703
- GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(FilterStackCall)) +
704
- channel_stack->call_stack_size;
705
-
706
- Arena* arena = channel->CreateArena();
707
- call = new (arena->Alloc(call_alloc_size)) FilterStackCall(arena, *args);
708
- DCHECK(FromC(call->c_ptr()) == call);
709
- DCHECK(FromCallStack(call->call_stack()) == call);
710
- *out_call = call->c_ptr();
711
- grpc_slice path = grpc_empty_slice();
712
- ScopedContext ctx(call);
713
- if (call->is_client()) {
714
- call->final_op_.client.status_details = nullptr;
715
- call->final_op_.client.status = nullptr;
716
- call->final_op_.client.error_string = nullptr;
717
- global_stats().IncrementClientCallsCreated();
718
- path = CSliceRef(args->path->c_slice());
719
- call->send_initial_metadata_.Set(HttpPathMetadata(),
720
- std::move(*args->path));
721
- if (args->authority.has_value()) {
722
- call->send_initial_metadata_.Set(HttpAuthorityMetadata(),
723
- std::move(*args->authority));
724
- }
725
- call->send_initial_metadata_.Set(
726
- GrpcRegisteredMethod(), reinterpret_cast<void*>(static_cast<uintptr_t>(
727
- args->registered_method)));
728
- channel_stack->stats_plugin_group->AddClientCallTracers(
729
- Slice(CSliceRef(path)), args->registered_method, call->context_);
730
- } else {
731
- global_stats().IncrementServerCallsCreated();
732
- call->final_op_.server.cancelled = nullptr;
733
- call->final_op_.server.core_server = args->server;
734
- // TODO(yashykt): In the future, we want to also enable stats and trace
735
- // collecting from when the call is created at the transport. The idea is
736
- // that the transport would create the call tracer and pass it in as part of
737
- // the metadata.
738
- // TODO(yijiem): OpenCensus and internal Census is still using this way to
739
- // set server call tracer. We need to refactor them to stats plugins
740
- // (including removing the client channel filters).
741
- if (args->server != nullptr &&
742
- args->server->server_call_tracer_factory() != nullptr) {
743
- auto* server_call_tracer =
744
- args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
745
- arena, args->server->channel_args());
746
- if (server_call_tracer != nullptr) {
747
- // Note that we are setting both
748
- // GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
749
- // GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
750
- // promise-based world, we would just a single tracer object for each
751
- // stack (call, subchannel_call, server_call.)
752
- call->ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
753
- server_call_tracer, nullptr);
754
- call->ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
755
- }
756
- }
757
- channel_stack->stats_plugin_group->AddServerCallTracers(call->context_);
758
- }
759
-
760
- Call* parent = Call::FromC(args->parent);
761
- if (parent != nullptr) {
762
- add_init_error(&error, absl_status_to_grpc_error(call->InitParent(
763
- parent, args->propagation_mask)));
764
- }
765
- // initial refcount dropped by grpc_call_unref
766
- grpc_call_element_args call_args = {
767
- call->call_stack(), args->server_transport_data,
768
- call->context_, path,
769
- call->start_time(), call->send_deadline(),
770
- call->arena(), &call->call_combiner_};
771
- add_init_error(&error, grpc_call_stack_init(channel_stack, 1, DestroyCall,
772
- call, &call_args));
773
- // Publish this call to parent only after the call stack has been initialized.
774
- if (parent != nullptr) {
775
- call->PublishToParent(parent);
776
- }
777
-
778
- if (!error.ok()) {
779
- call->CancelWithError(error);
780
- }
781
- if (args->cq != nullptr) {
782
- CHECK(args->pollset_set_alternative == nullptr)
783
- << "Only one of 'cq' and 'pollset_set_alternative' should be "
784
- "non-nullptr.";
785
- GRPC_CQ_INTERNAL_REF(args->cq, "bind");
786
- call->pollent_ =
787
- grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args->cq));
788
- }
789
- if (args->pollset_set_alternative != nullptr) {
790
- call->pollent_ = grpc_polling_entity_create_from_pollset_set(
791
- args->pollset_set_alternative);
792
- }
793
- if (!grpc_polling_entity_is_empty(&call->pollent_)) {
794
- grpc_call_stack_set_pollset_or_pollset_set(call->call_stack(),
795
- &call->pollent_);
796
- }
797
-
798
- if (call->is_client()) {
799
- channelz::ChannelNode* channelz_channel = channel->channelz_node();
800
- if (channelz_channel != nullptr) {
801
- channelz_channel->RecordCallStarted();
802
- }
803
- } else if (call->final_op_.server.core_server != nullptr) {
804
- channelz::ServerNode* channelz_node =
805
- call->final_op_.server.core_server->channelz_node();
806
- if (channelz_node != nullptr) {
807
- channelz_node->RecordCallStarted();
808
- }
809
- }
810
-
811
- if (args->send_deadline != Timestamp::InfFuture()) {
812
- call->UpdateDeadline(args->send_deadline);
813
- }
814
-
815
- CSliceUnref(path);
816
-
817
- return error;
818
- }
819
-
820
- void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) {
821
- CHECK(cq);
822
-
823
- if (grpc_polling_entity_pollset_set(&pollent_) != nullptr) {
824
- Crash("A pollset_set is already registered for this call.");
825
- }
826
- cq_ = cq;
827
- GRPC_CQ_INTERNAL_REF(cq, "bind");
828
- pollent_ = grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq));
829
- grpc_call_stack_set_pollset_or_pollset_set(call_stack(), &pollent_);
830
- }
831
-
832
- void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) {
833
- static_cast<FilterStackCall*>(call)->DeleteThis();
834
- }
835
-
836
- void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) {
837
- auto* c = static_cast<FilterStackCall*>(call);
838
- c->recv_initial_metadata_.Clear();
839
- c->recv_trailing_metadata_.Clear();
840
- c->receiving_slice_buffer_.reset();
841
- ParentCall* pc = c->parent_call();
842
- if (pc != nullptr) {
843
- pc->~ParentCall();
844
- }
845
- if (c->cq_) {
846
- GRPC_CQ_INTERNAL_UNREF(c->cq_, "bind");
847
- }
848
-
849
- grpc_error_handle status_error = c->status_error_.get();
850
- grpc_error_get_status(status_error, c->send_deadline(),
851
- &c->final_info_.final_status, nullptr, nullptr,
852
- &(c->final_info_.error_string));
853
- c->status_error_.set(absl::OkStatus());
854
- c->final_info_.stats.latency =
855
- gpr_cycle_counter_sub(gpr_get_cycle_counter(), c->start_time());
856
- grpc_call_stack_destroy(c->call_stack(), &c->final_info_,
857
- GRPC_CLOSURE_INIT(&c->release_call_, ReleaseCall, c,
858
- grpc_schedule_on_exec_ctx));
859
- }
860
-
861
- void FilterStackCall::ExternalUnref() {
862
- if (GPR_LIKELY(!ext_ref_.Unref())) return;
863
-
864
- ApplicationCallbackExecCtx callback_exec_ctx;
865
- ExecCtx exec_ctx;
866
-
867
- GRPC_API_TRACE("grpc_call_unref(c=%p)", 1, (this));
868
-
869
- MaybeUnpublishFromParent();
870
-
871
- CHECK(!destroy_called_);
872
- destroy_called_ = true;
873
- bool cancel = gpr_atm_acq_load(&received_final_op_atm_) == 0;
874
- if (cancel) {
875
- CancelWithError(absl::CancelledError());
876
- } else {
877
- // Unset the call combiner cancellation closure. This has the
878
- // effect of scheduling the previously set cancellation closure, if
879
- // any, so that it can release any internal references it may be
880
- // holding to the call stack.
881
- call_combiner_.SetNotifyOnCancel(nullptr);
882
- }
883
- InternalUnref("destroy");
884
- }
885
-
886
- // start_batch_closure points to a caller-allocated closure to be used
887
- // for entering the call combiner.
888
- void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch,
889
- grpc_closure* start_batch_closure) {
890
- // This is called via the call combiner to start sending a batch down
891
- // the filter stack.
892
- auto execute_batch_in_call_combiner = [](void* arg, grpc_error_handle) {
893
- grpc_transport_stream_op_batch* batch =
894
- static_cast<grpc_transport_stream_op_batch*>(arg);
895
- auto* call =
896
- static_cast<FilterStackCall*>(batch->handler_private.extra_arg);
897
- grpc_call_element* elem = call->call_elem(0);
898
- GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
899
- elem->filter->start_transport_stream_op_batch(elem, batch);
900
- };
901
- batch->handler_private.extra_arg = this;
902
- GRPC_CLOSURE_INIT(start_batch_closure, execute_batch_in_call_combiner, batch,
903
- grpc_schedule_on_exec_ctx);
904
- GRPC_CALL_COMBINER_START(call_combiner(), start_batch_closure,
905
- absl::OkStatus(), "executing batch");
906
- }
907
-
908
- namespace {
909
- struct CancelState {
910
- FilterStackCall* call;
911
- grpc_closure start_batch;
912
- grpc_closure finish_batch;
913
- };
914
- } // namespace
915
-
916
- // The on_complete callback used when sending a cancel_stream batch down
917
- // the filter stack. Yields the call combiner when the batch is done.
918
- static void done_termination(void* arg, grpc_error_handle /*error*/) {
919
- CancelState* state = static_cast<CancelState*>(arg);
920
- GRPC_CALL_COMBINER_STOP(state->call->call_combiner(),
921
- "on_complete for cancel_stream op");
922
- state->call->InternalUnref("termination");
923
- delete state;
924
- }
925
-
926
- void FilterStackCall::CancelWithError(grpc_error_handle error) {
927
- if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) {
928
- return;
929
- }
930
- if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
931
- gpr_log(GPR_INFO, "CancelWithError %s %s", is_client() ? "CLI" : "SVR",
932
- StatusToString(error).c_str());
933
- }
934
- ClearPeerString();
935
- InternalRef("termination");
936
- ResetDeadline();
937
- // Inform the call combiner of the cancellation, so that it can cancel
938
- // any in-flight asynchronous actions that may be holding the call
939
- // combiner. This ensures that the cancel_stream batch can be sent
940
- // down the filter stack in a timely manner.
941
- call_combiner_.Cancel(error);
942
- CancelState* state = new CancelState;
943
- state->call = this;
944
- GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,
945
- grpc_schedule_on_exec_ctx);
946
- grpc_transport_stream_op_batch* op =
947
- grpc_make_transport_stream_op(&state->finish_batch);
948
- op->cancel_stream = true;
949
- op->payload->cancel_stream.cancel_error = error;
950
- ExecuteBatch(op, &state->start_batch);
951
- }
952
-
953
- void FilterStackCall::SetFinalStatus(grpc_error_handle error) {
954
- if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) {
955
- gpr_log(GPR_INFO, "set_final_status %s %s", is_client() ? "CLI" : "SVR",
956
- StatusToString(error).c_str());
957
- }
958
- ResetDeadline();
959
- if (is_client()) {
960
- std::string status_details;
961
- grpc_error_get_status(error, send_deadline(), final_op_.client.status,
962
- &status_details, nullptr,
963
- final_op_.client.error_string);
964
- *final_op_.client.status_details =
965
- grpc_slice_from_cpp_string(std::move(status_details));
966
- status_error_.set(error);
967
- channelz::ChannelNode* channelz_channel = channel()->channelz_node();
968
- if (channelz_channel != nullptr) {
969
- if (*final_op_.client.status != GRPC_STATUS_OK) {
970
- channelz_channel->RecordCallFailed();
971
- } else {
972
- channelz_channel->RecordCallSucceeded();
973
- }
974
- }
975
- } else {
976
- *final_op_.server.cancelled =
977
- !error.ok() || !sent_server_trailing_metadata_;
978
- channelz::ServerNode* channelz_node =
979
- final_op_.server.core_server->channelz_node();
980
- if (channelz_node != nullptr) {
981
- if (*final_op_.server.cancelled || !status_error_.ok()) {
982
- channelz_node->RecordCallFailed();
983
- } else {
984
- channelz_node->RecordCallSucceeded();
985
- }
986
- }
987
- }
988
- }
989
-
990
- bool FilterStackCall::PrepareApplicationMetadata(size_t count,
991
- grpc_metadata* metadata,
992
- bool is_trailing) {
993
- grpc_metadata_batch* batch =
994
- is_trailing ? &send_trailing_metadata_ : &send_initial_metadata_;
995
- for (size_t i = 0; i < count; i++) {
996
- grpc_metadata* md = &metadata[i];
997
- if (!GRPC_LOG_IF_ERROR("validate_metadata",
998
- grpc_validate_header_key_is_legal(md->key))) {
999
- return false;
1000
- } else if (!grpc_is_binary_header_internal(md->key) &&
1001
- !GRPC_LOG_IF_ERROR(
1002
- "validate_metadata",
1003
- grpc_validate_header_nonbin_value_is_legal(md->value))) {
1004
- return false;
1005
- } else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
1006
- // HTTP2 hpack encoding has a maximum limit.
1007
- return false;
1008
- } else if (grpc_slice_str_cmp(md->key, "content-length") == 0) {
1009
- // Filter "content-length metadata"
1010
- continue;
1011
- }
1012
- batch->Append(StringViewFromSlice(md->key), Slice(CSliceRef(md->value)),
1013
- [md](absl::string_view error, const Slice& value) {
1014
- gpr_log(GPR_DEBUG, "Append error: %s",
1015
- absl::StrCat("key=", StringViewFromSlice(md->key),
1016
- " error=", error,
1017
- " value=", value.as_string_view())
1018
- .c_str());
1019
- });
1020
- }
1021
-
1022
- return true;
1023
- }
1024
-
1025
- namespace {
1026
- class PublishToAppEncoder {
1027
- public:
1028
- explicit PublishToAppEncoder(grpc_metadata_array* dest,
1029
- const grpc_metadata_batch* encoding,
1030
- bool is_client)
1031
- : dest_(dest), encoding_(encoding), is_client_(is_client) {}
1032
-
1033
- void Encode(const Slice& key, const Slice& value) {
1034
- Append(key.c_slice(), value.c_slice());
1035
- }
1036
-
1037
- // Catch anything that is not explicitly handled, and do not publish it to the
1038
- // application. If new metadata is added to a batch that needs to be
1039
- // published, it should be called out here.
1040
- template <typename Which>
1041
- void Encode(Which, const typename Which::ValueType&) {}
1042
-
1043
- void Encode(UserAgentMetadata, const Slice& slice) {
1044
- Append(UserAgentMetadata::key(), slice);
1045
- }
1046
-
1047
- void Encode(HostMetadata, const Slice& slice) {
1048
- Append(HostMetadata::key(), slice);
1049
- }
1050
-
1051
- void Encode(GrpcPreviousRpcAttemptsMetadata, uint32_t count) {
1052
- Append(GrpcPreviousRpcAttemptsMetadata::key(), count);
1053
- }
1054
-
1055
- void Encode(GrpcRetryPushbackMsMetadata, Duration count) {
1056
- Append(GrpcRetryPushbackMsMetadata::key(), count.millis());
1057
- }
1058
-
1059
- void Encode(LbTokenMetadata, const Slice& slice) {
1060
- Append(LbTokenMetadata::key(), slice);
1061
- }
1062
-
1063
- private:
1064
- void Append(absl::string_view key, int64_t value) {
1065
- Append(StaticSlice::FromStaticString(key).c_slice(),
1066
- Slice::FromInt64(value).c_slice());
1067
- }
1068
-
1069
- void Append(absl::string_view key, const Slice& value) {
1070
- Append(StaticSlice::FromStaticString(key).c_slice(), value.c_slice());
1071
- }
1072
-
1073
- void Append(grpc_slice key, grpc_slice value) {
1074
- if (dest_->count == dest_->capacity) {
1075
- Crash(absl::StrCat(
1076
- "Too many metadata entries: capacity=", dest_->capacity, " on ",
1077
- is_client_ ? "client" : "server", " encoding ", encoding_->count(),
1078
- " elements: ", encoding_->DebugString().c_str()));
1079
- }
1080
- auto* mdusr = &dest_->metadata[dest_->count++];
1081
- mdusr->key = key;
1082
- mdusr->value = value;
1083
- }
1084
-
1085
- grpc_metadata_array* const dest_;
1086
- const grpc_metadata_batch* const encoding_;
1087
- const bool is_client_;
1088
- };
1089
- } // namespace
1090
-
1091
- void FilterStackCall::PublishAppMetadata(grpc_metadata_batch* b,
1092
- bool is_trailing) {
1093
- if (b->count() == 0) return;
1094
- if (!is_client() && is_trailing) return;
1095
- if (is_trailing && buffered_metadata_[1] == nullptr) return;
1096
- grpc_metadata_array* dest;
1097
- dest = buffered_metadata_[is_trailing];
1098
- if (dest->count + b->count() > dest->capacity) {
1099
- dest->capacity =
1100
- std::max(dest->capacity + b->count(), dest->capacity * 3 / 2);
1101
- dest->metadata = static_cast<grpc_metadata*>(
1102
- gpr_realloc(dest->metadata, sizeof(grpc_metadata) * dest->capacity));
1103
- }
1104
- PublishToAppEncoder encoder(dest, b, is_client());
1105
- b->Encode(&encoder);
1106
- }
1107
-
1108
- void FilterStackCall::RecvInitialFilter(grpc_metadata_batch* b) {
1109
- ProcessIncomingInitialMetadata(*b);
1110
- PublishAppMetadata(b, false);
1111
- }
1112
-
1113
- void FilterStackCall::RecvTrailingFilter(grpc_metadata_batch* b,
1114
- grpc_error_handle batch_error) {
1115
- if (!batch_error.ok()) {
1116
- SetFinalStatus(batch_error);
1117
- } else {
1118
- absl::optional<grpc_status_code> grpc_status =
1119
- b->Take(GrpcStatusMetadata());
1120
- if (grpc_status.has_value()) {
1121
- grpc_status_code status_code = *grpc_status;
1122
- grpc_error_handle error;
1123
- if (status_code != GRPC_STATUS_OK) {
1124
- Slice peer = GetPeerString();
1125
- error = grpc_error_set_int(
1126
- GRPC_ERROR_CREATE(absl::StrCat("Error received from peer ",
1127
- peer.as_string_view())),
1128
- StatusIntProperty::kRpcStatus, static_cast<intptr_t>(status_code));
1129
- }
1130
- auto grpc_message = b->Take(GrpcMessageMetadata());
1131
- if (grpc_message.has_value()) {
1132
- error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage,
1133
- grpc_message->as_string_view());
1134
- } else if (!error.ok()) {
1135
- error = grpc_error_set_str(error, StatusStrProperty::kGrpcMessage, "");
1136
- }
1137
- SetFinalStatus(error);
1138
- } else if (!is_client()) {
1139
- SetFinalStatus(absl::OkStatus());
1140
- } else {
1141
- gpr_log(GPR_DEBUG,
1142
- "Received trailing metadata with no error and no status");
1143
- SetFinalStatus(grpc_error_set_int(GRPC_ERROR_CREATE("No status received"),
1144
- StatusIntProperty::kRpcStatus,
1145
- GRPC_STATUS_UNKNOWN));
1146
- }
1147
- }
1148
- PublishAppMetadata(b, true);
1149
- }
1150
-
1151
- namespace {
1152
- bool AreWriteFlagsValid(uint32_t flags) {
1153
- // check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
1154
- const uint32_t allowed_write_positions =
1155
- (GRPC_WRITE_USED_MASK | GRPC_WRITE_INTERNAL_USED_MASK);
1156
- const uint32_t invalid_positions = ~allowed_write_positions;
1157
- return !(flags & invalid_positions);
1158
- }
1159
-
1160
- bool AreInitialMetadataFlagsValid(uint32_t flags) {
1161
- // check that only bits in GRPC_WRITE_(INTERNAL?)_USED_MASK are set
1162
- uint32_t invalid_positions = ~GRPC_INITIAL_METADATA_USED_MASK;
1163
- return !(flags & invalid_positions);
1164
- }
1165
-
1166
- size_t BatchSlotForOp(grpc_op_type type) {
1167
- switch (type) {
1168
- case GRPC_OP_SEND_INITIAL_METADATA:
1169
- return 0;
1170
- case GRPC_OP_SEND_MESSAGE:
1171
- return 1;
1172
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
1173
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
1174
- return 2;
1175
- case GRPC_OP_RECV_INITIAL_METADATA:
1176
- return 3;
1177
- case GRPC_OP_RECV_MESSAGE:
1178
- return 4;
1179
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
1180
- case GRPC_OP_RECV_STATUS_ON_CLIENT:
1181
- return 5;
1182
- }
1183
- GPR_UNREACHABLE_CODE(return 123456789);
1184
- }
1185
- } // namespace
1186
-
1187
- FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl(
1188
- const grpc_op* ops) {
1189
- size_t slot_idx = BatchSlotForOp(ops[0].op);
1190
- BatchControl** pslot = &active_batches_[slot_idx];
1191
- BatchControl* bctl;
1192
- if (*pslot != nullptr) {
1193
- bctl = *pslot;
1194
- if (bctl->call_ != nullptr) {
1195
- return nullptr;
1196
- }
1197
- bctl->~BatchControl();
1198
- bctl->op_ = {};
1199
- new (&bctl->batch_error_) AtomicError();
1200
- } else {
1201
- bctl = arena()->New<BatchControl>();
1202
- *pslot = bctl;
1203
- }
1204
- bctl->call_ = this;
1205
- bctl->call_tracer_ = static_cast<CallTracerAnnotationInterface*>(
1206
- ContextGet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE));
1207
- bctl->op_.payload = &stream_op_payload_;
1208
- return bctl;
1209
- }
1210
-
1211
- void FilterStackCall::BatchControl::PostCompletion() {
1212
- FilterStackCall* call = call_;
1213
- grpc_error_handle error = batch_error_.get();
1214
-
1215
- if (IsCallStatusOverrideOnCancellationEnabled()) {
1216
- // On the client side, if final call status is already known (i.e if this op
1217
- // includes recv_trailing_metadata) and if the call status is known to be
1218
- // OK, then disregard the batch error to ensure call->receiving_buffer_ is
1219
- // not cleared.
1220
- if (op_.recv_trailing_metadata && call->is_client() &&
1221
- call->status_error_.ok()) {
1222
- error = absl::OkStatus();
1223
- }
1224
- }
1225
-
1226
- if (grpc_call_trace.enabled()) {
1227
- gpr_log(GPR_DEBUG, "tag:%p batch_error=%s op:%s",
1228
- completion_data_.notify_tag.tag, error.ToString().c_str(),
1229
- grpc_transport_stream_op_batch_string(&op_, false).c_str());
1230
- }
1231
-
1232
- if (op_.send_initial_metadata) {
1233
- call->send_initial_metadata_.Clear();
1234
- }
1235
- if (op_.send_message) {
1236
- if (op_.payload->send_message.stream_write_closed && error.ok()) {
1237
- error = grpc_error_add_child(
1238
- error, GRPC_ERROR_CREATE(
1239
- "Attempt to send message after stream was closed."));
1240
- }
1241
- call->sending_message_ = false;
1242
- call->send_slice_buffer_.Clear();
1243
- }
1244
- if (op_.send_trailing_metadata) {
1245
- call->send_trailing_metadata_.Clear();
1246
- }
1247
-
1248
- if (!error.ok() && op_.recv_message && *call->receiving_buffer_ != nullptr) {
1249
- grpc_byte_buffer_destroy(*call->receiving_buffer_);
1250
- *call->receiving_buffer_ = nullptr;
1251
- }
1252
- if (op_.recv_trailing_metadata) {
1253
- // propagate cancellation to any interested children
1254
- gpr_atm_rel_store(&call->received_final_op_atm_, 1);
1255
- call->PropagateCancellationToChildren();
1256
- error = absl::OkStatus();
1257
- }
1258
- batch_error_.set(absl::OkStatus());
1259
-
1260
- if (completion_data_.notify_tag.is_closure) {
1261
- call_ = nullptr;
1262
- GrpcClosure::Run(
1263
- DEBUG_LOCATION,
1264
- static_cast<grpc_closure*>(completion_data_.notify_tag.tag), error);
1265
- call->InternalUnref("completion");
1266
- } else {
1267
- grpc_cq_end_op(
1268
- call->cq_, completion_data_.notify_tag.tag, error,
1269
- [](void* user_data, grpc_cq_completion* /*storage*/) {
1270
- BatchControl* bctl = static_cast<BatchControl*>(user_data);
1271
- Call* call = bctl->call_;
1272
- bctl->call_ = nullptr;
1273
- call->InternalUnref("completion");
1274
- },
1275
- this, &completion_data_.cq_completion);
1276
- }
1277
- }
1278
-
1279
- void FilterStackCall::BatchControl::FinishStep(PendingOp op) {
1280
- if (GPR_UNLIKELY(completed_batch_step(op))) {
1281
- PostCompletion();
1282
- }
1283
- }
1284
-
1285
- void FilterStackCall::BatchControl::ProcessDataAfterMetadata() {
1286
- FilterStackCall* call = call_;
1287
- if (!call->receiving_slice_buffer_.has_value()) {
1288
- *call->receiving_buffer_ = nullptr;
1289
- call->receiving_message_ = false;
1290
- FinishStep(PendingOp::kRecvMessage);
1291
- } else {
1292
- call->NoteLastMessageFlags(call->receiving_stream_flags_);
1293
- if ((call->receiving_stream_flags_ & GRPC_WRITE_INTERNAL_COMPRESS) &&
1294
- (call->incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
1295
- *call->receiving_buffer_ = grpc_raw_compressed_byte_buffer_create(
1296
- nullptr, 0, call->incoming_compression_algorithm());
1297
- } else {
1298
- *call->receiving_buffer_ = grpc_raw_byte_buffer_create(nullptr, 0);
1299
- }
1300
- grpc_slice_buffer_move_into(
1301
- call->receiving_slice_buffer_->c_slice_buffer(),
1302
- &(*call->receiving_buffer_)->data.raw.slice_buffer);
1303
- call->receiving_message_ = false;
1304
- call->receiving_slice_buffer_.reset();
1305
- FinishStep(PendingOp::kRecvMessage);
1306
- }
1307
- }
1308
-
1309
- void FilterStackCall::BatchControl::ReceivingStreamReady(
1310
- grpc_error_handle error) {
1311
- if (grpc_call_trace.enabled()) {
1312
- gpr_log(GPR_DEBUG,
1313
- "tag:%p ReceivingStreamReady error=%s "
1314
- "receiving_slice_buffer.has_value=%d recv_state=%" PRIdPTR,
1315
- completion_data_.notify_tag.tag, error.ToString().c_str(),
1316
- call_->receiving_slice_buffer_.has_value(),
1317
- gpr_atm_no_barrier_load(&call_->recv_state_));
1318
- }
1319
- FilterStackCall* call = call_;
1320
- if (!error.ok()) {
1321
- call->receiving_slice_buffer_.reset();
1322
- if (batch_error_.ok()) {
1323
- batch_error_.set(error);
1324
- }
1325
- call->CancelWithError(error);
1326
- }
1327
- // If recv_state is kRecvNone, we will save the batch_control
1328
- // object with rel_cas, and will not use it after the cas. Its corresponding
1329
- // acq_load is in receiving_initial_metadata_ready()
1330
- if (!error.ok() || !call->receiving_slice_buffer_.has_value() ||
1331
- !gpr_atm_rel_cas(&call->recv_state_, kRecvNone,
1332
- reinterpret_cast<gpr_atm>(this))) {
1333
- ProcessDataAfterMetadata();
1334
- }
1335
- }
1336
-
1337
- void FilterStackCall::BatchControl::ReceivingInitialMetadataReady(
1338
- grpc_error_handle error) {
1339
- FilterStackCall* call = call_;
1340
-
1341
- GRPC_CALL_COMBINER_STOP(call->call_combiner(), "recv_initial_metadata_ready");
1342
-
1343
- if (error.ok()) {
1344
- grpc_metadata_batch* md = &call->recv_initial_metadata_;
1345
- call->RecvInitialFilter(md);
1346
-
1347
- absl::optional<Timestamp> deadline = md->get(GrpcTimeoutMetadata());
1348
- if (deadline.has_value() && !call->is_client()) {
1349
- call_->set_send_deadline(*deadline);
1350
- }
1351
- } else {
1352
- if (batch_error_.ok()) {
1353
- batch_error_.set(error);
1354
- }
1355
- call->CancelWithError(error);
1356
- }
384
+ } // namespace grpc_core
1357
385
 
1358
- grpc_closure* saved_rsr_closure = nullptr;
1359
- while (true) {
1360
- gpr_atm rsr_bctlp = gpr_atm_acq_load(&call->recv_state_);
1361
- // Should only receive initial metadata once
1362
- CHECK_NE(rsr_bctlp, 1);
1363
- if (rsr_bctlp == 0) {
1364
- // We haven't seen initial metadata and messages before, thus initial
1365
- // metadata is received first.
1366
- // no_barrier_cas is used, as this function won't access the batch_control
1367
- // object saved by receiving_stream_ready() if the initial metadata is
1368
- // received first.
1369
- if (gpr_atm_no_barrier_cas(&call->recv_state_, kRecvNone,
1370
- kRecvInitialMetadataFirst)) {
1371
- break;
1372
- }
1373
- } else {
1374
- // Already received messages
1375
- saved_rsr_closure = GRPC_CLOSURE_CREATE(
1376
- [](void* bctl, grpc_error_handle error) {
1377
- static_cast<BatchControl*>(bctl)->ReceivingStreamReady(error);
1378
- },
1379
- reinterpret_cast<BatchControl*>(rsr_bctlp),
1380
- grpc_schedule_on_exec_ctx);
1381
- // No need to modify recv_state
1382
- break;
1383
- }
1384
- }
1385
- if (saved_rsr_closure != nullptr) {
1386
- GrpcClosure::Run(DEBUG_LOCATION, saved_rsr_closure, error);
1387
- }
386
+ ///////////////////////////////////////////////////////////////////////////////
387
+ // C-based API
1388
388
 
1389
- FinishStep(PendingOp::kRecvInitialMetadata);
389
+ void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
390
+ grpc_core::ExecCtx exec_ctx;
391
+ return grpc_core::Call::FromC(call)->arena()->Alloc(size);
1390
392
  }
1391
393
 
1392
- void FilterStackCall::BatchControl::ReceivingTrailingMetadataReady(
1393
- grpc_error_handle error) {
1394
- GRPC_CALL_COMBINER_STOP(call_->call_combiner(),
1395
- "recv_trailing_metadata_ready");
1396
- grpc_metadata_batch* md = &call_->recv_trailing_metadata_;
1397
- call_->RecvTrailingFilter(md, error);
1398
- FinishStep(PendingOp::kRecvTrailingMetadata);
394
+ void grpc_call_set_completion_queue(grpc_call* call,
395
+ grpc_completion_queue* cq) {
396
+ grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
1399
397
  }
1400
398
 
1401
- void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) {
1402
- GRPC_CALL_COMBINER_STOP(call_->call_combiner(), "on_complete");
1403
- if (batch_error_.ok()) {
1404
- batch_error_.set(error);
1405
- }
1406
- if (!error.ok()) {
1407
- call_->CancelWithError(error);
1408
- }
1409
- FinishStep(PendingOp::kSends);
1410
- }
399
+ void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
1411
400
 
1412
- namespace {
1413
- void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag,
1414
- bool is_notify_tag_closure) {
1415
- if (!is_notify_tag_closure) {
1416
- CHECK(grpc_cq_begin_op(cq, notify_tag));
1417
- grpc_cq_end_op(
1418
- cq, notify_tag, absl::OkStatus(),
1419
- [](void*, grpc_cq_completion* completion) { gpr_free(completion); },
1420
- nullptr,
1421
- static_cast<grpc_cq_completion*>(
1422
- gpr_malloc(sizeof(grpc_cq_completion))));
1423
- } else {
1424
- Closure::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(notify_tag),
1425
- absl::OkStatus());
1426
- }
1427
- }
1428
- } // namespace
1429
-
1430
- grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops,
1431
- void* notify_tag,
1432
- bool is_notify_tag_closure) {
1433
- size_t i;
1434
- const grpc_op* op;
1435
- BatchControl* bctl;
1436
- grpc_call_error error = GRPC_CALL_OK;
1437
- grpc_transport_stream_op_batch* stream_op;
1438
- grpc_transport_stream_op_batch_payload* stream_op_payload;
1439
- uint32_t seen_ops = 0;
1440
- intptr_t pending_ops = 0;
1441
-
1442
- for (i = 0; i < nops; i++) {
1443
- if (seen_ops & (1u << ops[i].op)) {
1444
- return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1445
- }
1446
- seen_ops |= (1u << ops[i].op);
1447
- }
1448
-
1449
- if (!is_client() &&
1450
- (seen_ops & (1u << GRPC_OP_SEND_STATUS_FROM_SERVER)) != 0 &&
1451
- (seen_ops & (1u << GRPC_OP_RECV_MESSAGE)) != 0) {
1452
- gpr_log(GPR_ERROR,
1453
- "******************* SEND_STATUS WITH RECV_MESSAGE "
1454
- "*******************");
1455
- return GRPC_CALL_ERROR;
1456
- }
1457
-
1458
- GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops);
1459
-
1460
- if (nops == 0) {
1461
- EndOpImmediately(cq_, notify_tag, is_notify_tag_closure);
1462
- error = GRPC_CALL_OK;
1463
- goto done;
1464
- }
1465
-
1466
- bctl = ReuseOrAllocateBatchControl(ops);
1467
- if (bctl == nullptr) {
1468
- return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1469
- }
1470
- bctl->completion_data_.notify_tag.tag = notify_tag;
1471
- bctl->completion_data_.notify_tag.is_closure =
1472
- static_cast<uint8_t>(is_notify_tag_closure != 0);
1473
-
1474
- stream_op = &bctl->op_;
1475
- stream_op_payload = &stream_op_payload_;
1476
-
1477
- // rewrite batch ops into a transport op
1478
- for (i = 0; i < nops; i++) {
1479
- op = &ops[i];
1480
- if (op->reserved != nullptr) {
1481
- error = GRPC_CALL_ERROR;
1482
- goto done_with_error;
1483
- }
1484
- switch (op->op) {
1485
- case GRPC_OP_SEND_INITIAL_METADATA: {
1486
- // Flag validation: currently allow no flags
1487
- if (!AreInitialMetadataFlagsValid(op->flags)) {
1488
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1489
- goto done_with_error;
1490
- }
1491
- if (sent_initial_metadata_) {
1492
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1493
- goto done_with_error;
1494
- }
1495
- if (op->data.send_initial_metadata.count > INT_MAX) {
1496
- error = GRPC_CALL_ERROR_INVALID_METADATA;
1497
- goto done_with_error;
1498
- }
1499
- stream_op->send_initial_metadata = true;
1500
- sent_initial_metadata_ = true;
1501
- if (!PrepareApplicationMetadata(op->data.send_initial_metadata.count,
1502
- op->data.send_initial_metadata.metadata,
1503
- false)) {
1504
- error = GRPC_CALL_ERROR_INVALID_METADATA;
1505
- goto done_with_error;
1506
- }
1507
- PrepareOutgoingInitialMetadata(*op, send_initial_metadata_);
1508
- // TODO(ctiller): just make these the same variable?
1509
- if (is_client() && send_deadline() != Timestamp::InfFuture()) {
1510
- send_initial_metadata_.Set(GrpcTimeoutMetadata(), send_deadline());
1511
- }
1512
- if (is_client()) {
1513
- send_initial_metadata_.Set(
1514
- WaitForReady(),
1515
- WaitForReady::ValueType{
1516
- (op->flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
1517
- (op->flags &
1518
- GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
1519
- }
1520
- stream_op_payload->send_initial_metadata.send_initial_metadata =
1521
- &send_initial_metadata_;
1522
- pending_ops |= PendingOpMask(PendingOp::kSends);
1523
- break;
1524
- }
1525
- case GRPC_OP_SEND_MESSAGE: {
1526
- if (!AreWriteFlagsValid(op->flags)) {
1527
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1528
- goto done_with_error;
1529
- }
1530
- if (op->data.send_message.send_message == nullptr) {
1531
- error = GRPC_CALL_ERROR_INVALID_MESSAGE;
1532
- goto done_with_error;
1533
- }
1534
- if (sending_message_) {
1535
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1536
- goto done_with_error;
1537
- }
1538
- uint32_t flags = op->flags;
1539
- // If the outgoing buffer is already compressed, mark it as so in the
1540
- // flags. These will be picked up by the compression filter and further
1541
- // (wasteful) attempts at compression skipped.
1542
- if (op->data.send_message.send_message->data.raw.compression >
1543
- GRPC_COMPRESS_NONE) {
1544
- flags |= GRPC_WRITE_INTERNAL_COMPRESS;
1545
- }
1546
- stream_op->send_message = true;
1547
- sending_message_ = true;
1548
- send_slice_buffer_.Clear();
1549
- grpc_slice_buffer_move_into(
1550
- &op->data.send_message.send_message->data.raw.slice_buffer,
1551
- send_slice_buffer_.c_slice_buffer());
1552
- stream_op_payload->send_message.flags = flags;
1553
- stream_op_payload->send_message.send_message = &send_slice_buffer_;
1554
- pending_ops |= PendingOpMask(PendingOp::kSends);
1555
- break;
1556
- }
1557
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT: {
1558
- // Flag validation: currently allow no flags
1559
- if (op->flags != 0) {
1560
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1561
- goto done_with_error;
1562
- }
1563
- if (!is_client()) {
1564
- error = GRPC_CALL_ERROR_NOT_ON_SERVER;
1565
- goto done_with_error;
1566
- }
1567
- if (sent_final_op_) {
1568
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1569
- goto done_with_error;
1570
- }
1571
- stream_op->send_trailing_metadata = true;
1572
- sent_final_op_ = true;
1573
- stream_op_payload->send_trailing_metadata.send_trailing_metadata =
1574
- &send_trailing_metadata_;
1575
- pending_ops |= PendingOpMask(PendingOp::kSends);
1576
- break;
1577
- }
1578
- case GRPC_OP_SEND_STATUS_FROM_SERVER: {
1579
- // Flag validation: currently allow no flags
1580
- if (op->flags != 0) {
1581
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1582
- goto done_with_error;
1583
- }
1584
- if (is_client()) {
1585
- error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
1586
- goto done_with_error;
1587
- }
1588
- if (sent_final_op_) {
1589
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1590
- goto done_with_error;
1591
- }
1592
- if (op->data.send_status_from_server.trailing_metadata_count >
1593
- INT_MAX) {
1594
- error = GRPC_CALL_ERROR_INVALID_METADATA;
1595
- goto done_with_error;
1596
- }
1597
- stream_op->send_trailing_metadata = true;
1598
- sent_final_op_ = true;
1599
-
1600
- if (!PrepareApplicationMetadata(
1601
- op->data.send_status_from_server.trailing_metadata_count,
1602
- op->data.send_status_from_server.trailing_metadata, true)) {
1603
- error = GRPC_CALL_ERROR_INVALID_METADATA;
1604
- goto done_with_error;
1605
- }
1606
-
1607
- grpc_error_handle status_error =
1608
- op->data.send_status_from_server.status == GRPC_STATUS_OK
1609
- ? absl::OkStatus()
1610
- : grpc_error_set_int(
1611
- GRPC_ERROR_CREATE("Server returned error"),
1612
- StatusIntProperty::kRpcStatus,
1613
- static_cast<intptr_t>(
1614
- op->data.send_status_from_server.status));
1615
- if (op->data.send_status_from_server.status_details != nullptr) {
1616
- send_trailing_metadata_.Set(
1617
- GrpcMessageMetadata(),
1618
- Slice(grpc_slice_copy(
1619
- *op->data.send_status_from_server.status_details)));
1620
- if (!status_error.ok()) {
1621
- status_error = grpc_error_set_str(
1622
- status_error, StatusStrProperty::kGrpcMessage,
1623
- StringViewFromSlice(
1624
- *op->data.send_status_from_server.status_details));
1625
- }
1626
- }
1627
-
1628
- status_error_.set(status_error);
1629
-
1630
- send_trailing_metadata_.Set(GrpcStatusMetadata(),
1631
- op->data.send_status_from_server.status);
1632
-
1633
- // Ignore any te metadata key value pairs specified.
1634
- send_trailing_metadata_.Remove(TeMetadata());
1635
- stream_op_payload->send_trailing_metadata.send_trailing_metadata =
1636
- &send_trailing_metadata_;
1637
- stream_op_payload->send_trailing_metadata.sent =
1638
- &sent_server_trailing_metadata_;
1639
- pending_ops |= PendingOpMask(PendingOp::kSends);
1640
- break;
1641
- }
1642
- case GRPC_OP_RECV_INITIAL_METADATA: {
1643
- // Flag validation: currently allow no flags
1644
- if (op->flags != 0) {
1645
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1646
- goto done_with_error;
1647
- }
1648
- if (received_initial_metadata_) {
1649
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1650
- goto done_with_error;
1651
- }
1652
- received_initial_metadata_ = true;
1653
- buffered_metadata_[0] =
1654
- op->data.recv_initial_metadata.recv_initial_metadata;
1655
- GRPC_CLOSURE_INIT(
1656
- &receiving_initial_metadata_ready_,
1657
- [](void* bctl, grpc_error_handle error) {
1658
- static_cast<BatchControl*>(bctl)->ReceivingInitialMetadataReady(
1659
- error);
1660
- },
1661
- bctl, grpc_schedule_on_exec_ctx);
1662
- stream_op->recv_initial_metadata = true;
1663
- stream_op_payload->recv_initial_metadata.recv_initial_metadata =
1664
- &recv_initial_metadata_;
1665
- stream_op_payload->recv_initial_metadata.recv_initial_metadata_ready =
1666
- &receiving_initial_metadata_ready_;
1667
- if (is_client()) {
1668
- stream_op_payload->recv_initial_metadata.trailing_metadata_available =
1669
- &is_trailers_only_;
1670
- }
1671
- pending_ops |= PendingOpMask(PendingOp::kRecvInitialMetadata);
1672
- break;
1673
- }
1674
- case GRPC_OP_RECV_MESSAGE: {
1675
- // Flag validation: currently allow no flags
1676
- if (op->flags != 0) {
1677
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1678
- goto done_with_error;
1679
- }
1680
- if (receiving_message_) {
1681
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1682
- goto done_with_error;
1683
- }
1684
- receiving_message_ = true;
1685
- stream_op->recv_message = true;
1686
- receiving_slice_buffer_.reset();
1687
- receiving_buffer_ = op->data.recv_message.recv_message;
1688
- stream_op_payload->recv_message.recv_message = &receiving_slice_buffer_;
1689
- receiving_stream_flags_ = 0;
1690
- stream_op_payload->recv_message.flags = &receiving_stream_flags_;
1691
- stream_op_payload->recv_message.call_failed_before_recv_message =
1692
- &call_failed_before_recv_message_;
1693
- GRPC_CLOSURE_INIT(
1694
- &receiving_stream_ready_,
1695
- [](void* bctlp, grpc_error_handle error) {
1696
- auto* bctl = static_cast<BatchControl*>(bctlp);
1697
- auto* call = bctl->call_;
1698
- // Yields the call combiner before processing the received
1699
- // message.
1700
- GRPC_CALL_COMBINER_STOP(call->call_combiner(),
1701
- "recv_message_ready");
1702
- bctl->ReceivingStreamReady(error);
1703
- },
1704
- bctl, grpc_schedule_on_exec_ctx);
1705
- stream_op_payload->recv_message.recv_message_ready =
1706
- &receiving_stream_ready_;
1707
- pending_ops |= PendingOpMask(PendingOp::kRecvMessage);
1708
- break;
1709
- }
1710
- case GRPC_OP_RECV_STATUS_ON_CLIENT: {
1711
- // Flag validation: currently allow no flags
1712
- if (op->flags != 0) {
1713
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1714
- goto done_with_error;
1715
- }
1716
- if (!is_client()) {
1717
- error = GRPC_CALL_ERROR_NOT_ON_SERVER;
1718
- goto done_with_error;
1719
- }
1720
- if (requested_final_op_) {
1721
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1722
- goto done_with_error;
1723
- }
1724
- requested_final_op_ = true;
1725
- buffered_metadata_[1] =
1726
- op->data.recv_status_on_client.trailing_metadata;
1727
- final_op_.client.status = op->data.recv_status_on_client.status;
1728
- final_op_.client.status_details =
1729
- op->data.recv_status_on_client.status_details;
1730
- final_op_.client.error_string =
1731
- op->data.recv_status_on_client.error_string;
1732
- stream_op->recv_trailing_metadata = true;
1733
- stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
1734
- &recv_trailing_metadata_;
1735
- stream_op_payload->recv_trailing_metadata.collect_stats =
1736
- &final_info_.stats.transport_stream_stats;
1737
- GRPC_CLOSURE_INIT(
1738
- &receiving_trailing_metadata_ready_,
1739
- [](void* bctl, grpc_error_handle error) {
1740
- static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
1741
- error);
1742
- },
1743
- bctl, grpc_schedule_on_exec_ctx);
1744
- stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
1745
- &receiving_trailing_metadata_ready_;
1746
- pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
1747
- break;
1748
- }
1749
- case GRPC_OP_RECV_CLOSE_ON_SERVER: {
1750
- // Flag validation: currently allow no flags
1751
- if (op->flags != 0) {
1752
- error = GRPC_CALL_ERROR_INVALID_FLAGS;
1753
- goto done_with_error;
1754
- }
1755
- if (is_client()) {
1756
- error = GRPC_CALL_ERROR_NOT_ON_CLIENT;
1757
- goto done_with_error;
1758
- }
1759
- if (requested_final_op_) {
1760
- error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
1761
- goto done_with_error;
1762
- }
1763
- requested_final_op_ = true;
1764
- final_op_.server.cancelled = op->data.recv_close_on_server.cancelled;
1765
- stream_op->recv_trailing_metadata = true;
1766
- stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
1767
- &recv_trailing_metadata_;
1768
- stream_op_payload->recv_trailing_metadata.collect_stats =
1769
- &final_info_.stats.transport_stream_stats;
1770
- GRPC_CLOSURE_INIT(
1771
- &receiving_trailing_metadata_ready_,
1772
- [](void* bctl, grpc_error_handle error) {
1773
- static_cast<BatchControl*>(bctl)->ReceivingTrailingMetadataReady(
1774
- error);
1775
- },
1776
- bctl, grpc_schedule_on_exec_ctx);
1777
- stream_op_payload->recv_trailing_metadata.recv_trailing_metadata_ready =
1778
- &receiving_trailing_metadata_ready_;
1779
- pending_ops |= PendingOpMask(PendingOp::kRecvTrailingMetadata);
1780
- break;
1781
- }
1782
- }
1783
- }
1784
-
1785
- InternalRef("completion");
1786
- if (!is_notify_tag_closure) {
1787
- CHECK(grpc_cq_begin_op(cq_, notify_tag));
1788
- }
1789
- bctl->set_pending_ops(pending_ops);
1790
-
1791
- if (pending_ops & PendingOpMask(PendingOp::kSends)) {
1792
- GRPC_CLOSURE_INIT(
1793
- &bctl->finish_batch_,
1794
- [](void* bctl, grpc_error_handle error) {
1795
- static_cast<BatchControl*>(bctl)->FinishBatch(error);
1796
- },
1797
- bctl, grpc_schedule_on_exec_ctx);
1798
- stream_op->on_complete = &bctl->finish_batch_;
1799
- }
1800
-
1801
- if (grpc_call_trace.enabled()) {
1802
- gpr_log(GPR_DEBUG, "BATCH:%p START:%s BATCH:%s (tag:%p)", bctl,
1803
- PendingOpString(pending_ops).c_str(),
1804
- grpc_transport_stream_op_batch_string(stream_op, false).c_str(),
1805
- bctl->completion_data_.notify_tag.tag);
1806
- }
1807
- ExecuteBatch(stream_op, &bctl->start_batch_);
1808
-
1809
- done:
1810
- return error;
1811
-
1812
- done_with_error:
1813
- // reverse any mutations that occurred
1814
- if (stream_op->send_initial_metadata) {
1815
- sent_initial_metadata_ = false;
1816
- send_initial_metadata_.Clear();
1817
- }
1818
- if (stream_op->send_message) {
1819
- sending_message_ = false;
1820
- }
1821
- if (stream_op->send_trailing_metadata) {
1822
- sent_final_op_ = false;
1823
- send_trailing_metadata_.Clear();
1824
- }
1825
- if (stream_op->recv_initial_metadata) {
1826
- received_initial_metadata_ = false;
1827
- }
1828
- if (stream_op->recv_message) {
1829
- receiving_message_ = false;
1830
- }
1831
- if (stream_op->recv_trailing_metadata) {
1832
- requested_final_op_ = false;
1833
- }
1834
- goto done;
1835
- }
1836
-
1837
- void FilterStackCall::ContextSet(grpc_context_index elem, void* value,
1838
- void (*destroy)(void*)) {
1839
- if (context_[elem].destroy) {
1840
- context_[elem].destroy(context_[elem].value);
1841
- }
1842
- context_[elem].value = value;
1843
- context_[elem].destroy = destroy;
1844
- }
1845
-
1846
- ///////////////////////////////////////////////////////////////////////////////
1847
- // Metadata validation helpers
1848
-
1849
- namespace {
1850
- bool ValidateMetadata(size_t count, grpc_metadata* metadata) {
1851
- if (count > INT_MAX) {
1852
- return false;
1853
- }
1854
- for (size_t i = 0; i < count; i++) {
1855
- grpc_metadata* md = &metadata[i];
1856
- if (!GRPC_LOG_IF_ERROR("validate_metadata",
1857
- grpc_validate_header_key_is_legal(md->key))) {
1858
- return false;
1859
- } else if (!grpc_is_binary_header_internal(md->key) &&
1860
- !GRPC_LOG_IF_ERROR(
1861
- "validate_metadata",
1862
- grpc_validate_header_nonbin_value_is_legal(md->value))) {
1863
- return false;
1864
- } else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) {
1865
- // HTTP2 hpack encoding has a maximum limit.
1866
- return false;
1867
- }
1868
- }
1869
- return true;
1870
- }
1871
- } // namespace
1872
-
1873
- ///////////////////////////////////////////////////////////////////////////////
1874
- // PromiseBasedCall
1875
- // Will be folded into Call once the promise conversion is done
1876
-
1877
- class BasicPromiseBasedCall : public Call, public Party {
1878
- public:
1879
- using Call::arena;
1880
-
1881
- BasicPromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
1882
- uint32_t initial_internal_refs,
1883
- const grpc_call_create_args& args)
1884
- : Call(arena, args.server_transport_data == nullptr, args.send_deadline,
1885
- args.channel->Ref()),
1886
- Party(initial_internal_refs),
1887
- external_refs_(initial_external_refs),
1888
- cq_(args.cq) {
1889
- if (args.cq != nullptr) {
1890
- GRPC_CQ_INTERNAL_REF(args.cq, "bind");
1891
- }
1892
- context_[GRPC_CONTEXT_CALL].value = this;
1893
- }
1894
-
1895
- ~BasicPromiseBasedCall() override {
1896
- if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind");
1897
- for (int i = 0; i < GRPC_CONTEXT_COUNT; i++) {
1898
- if (context_[i].destroy) {
1899
- context_[i].destroy(context_[i].value);
1900
- }
1901
- }
1902
- }
1903
-
1904
- virtual void OrphanCall() = 0;
1905
-
1906
- virtual ServerCallContext* server_call_context() { return nullptr; }
1907
- void SetCompletionQueue(grpc_completion_queue* cq) final {
1908
- cq_ = cq;
1909
- GRPC_CQ_INTERNAL_REF(cq, "bind");
1910
- }
1911
-
1912
- // Implementation of call refcounting: move this to DualRefCounted once we
1913
- // don't need to maintain FilterStackCall compatibility
1914
- void ExternalRef() final {
1915
- if (external_refs_.fetch_add(1, std::memory_order_relaxed) == 0) {
1916
- InternalRef("external");
1917
- }
1918
- }
1919
- void ExternalUnref() final {
1920
- if (external_refs_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
1921
- OrphanCall();
1922
- InternalUnref("external");
1923
- }
1924
- }
1925
- void InternalRef(const char* reason) final {
1926
- if (grpc_call_refcount_trace.enabled()) {
1927
- gpr_log(GPR_DEBUG, "INTERNAL_REF:%p:%s", this, reason);
1928
- }
1929
- Party::IncrementRefCount();
1930
- }
1931
- void InternalUnref(const char* reason) final {
1932
- if (grpc_call_refcount_trace.enabled()) {
1933
- gpr_log(GPR_DEBUG, "INTERNAL_UNREF:%p:%s", this, reason);
1934
- }
1935
- Party::Unref();
1936
- }
1937
-
1938
- void RunInContext(absl::AnyInvocable<void()> fn) {
1939
- Spawn(
1940
- "run_in_context",
1941
- [fn = std::move(fn)]() mutable {
1942
- fn();
1943
- return Empty{};
1944
- },
1945
- [](Empty) {});
1946
- }
1947
-
1948
- void ContextSet(grpc_context_index elem, void* value,
1949
- void (*destroy)(void*)) final {
1950
- if (context_[elem].destroy != nullptr) {
1951
- context_[elem].destroy(context_[elem].value);
1952
- }
1953
- context_[elem].value = value;
1954
- context_[elem].destroy = destroy;
1955
- }
1956
-
1957
- void* ContextGet(grpc_context_index elem) const final {
1958
- return context_[elem].value;
1959
- }
1960
-
1961
- // Accept the stats from the context (call once we have proof the transport is
1962
- // done with them).
1963
- void AcceptTransportStatsFromContext() {
1964
- final_stats_ = *call_context_.call_stats();
1965
- }
1966
-
1967
- // This should return nullptr for the promise stack (and alternative means
1968
- // for that functionality be invented)
1969
- grpc_call_stack* call_stack() final { return nullptr; }
1970
-
1971
- virtual RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs) {
1972
- Crash("Not implemented");
1973
- }
1974
-
1975
- protected:
1976
- class ScopedContext
1977
- : public ScopedActivity,
1978
- public promise_detail::Context<Arena>,
1979
- public promise_detail::Context<grpc_call_context_element>,
1980
- public promise_detail::Context<CallContext>,
1981
- public promise_detail::Context<CallFinalization> {
1982
- public:
1983
- explicit ScopedContext(BasicPromiseBasedCall* call)
1984
- : ScopedActivity(call),
1985
- promise_detail::Context<Arena>(call->arena()),
1986
- promise_detail::Context<grpc_call_context_element>(call->context_),
1987
- promise_detail::Context<CallContext>(&call->call_context_),
1988
- promise_detail::Context<CallFinalization>(&call->finalization_) {}
1989
- };
1990
-
1991
- grpc_call_context_element* context() { return context_; }
1992
-
1993
- grpc_completion_queue* cq() { return cq_; }
1994
-
1995
- // At the end of the call run any finalization actions.
1996
- void SetFinalizationStatus(grpc_status_code status, Slice status_details) {
1997
- final_message_ = std::move(status_details);
1998
- final_status_ = status;
1999
- }
2000
-
2001
- grpc_event_engine::experimental::EventEngine* event_engine() const override {
2002
- return channel()->event_engine();
2003
- }
2004
-
2005
- private:
2006
- void PartyOver() final {
2007
- {
2008
- ScopedContext ctx(this);
2009
- std::string message;
2010
- grpc_call_final_info final_info;
2011
- final_info.stats = final_stats_;
2012
- final_info.final_status = final_status_;
2013
- // TODO(ctiller): change type here so we don't need to copy this string.
2014
- final_info.error_string = nullptr;
2015
- if (!final_message_.empty()) {
2016
- message = std::string(final_message_.begin(), final_message_.end());
2017
- final_info.error_string = message.c_str();
2018
- }
2019
- final_info.stats.latency =
2020
- gpr_cycle_counter_sub(gpr_get_cycle_counter(), start_time());
2021
- finalization_.Run(&final_info);
2022
- CancelRemainingParticipants();
2023
- arena()->DestroyManagedNewObjects();
2024
- }
2025
- DeleteThis();
2026
- }
2027
-
2028
- // Double refcounted for now: party owns the internal refcount, we track the
2029
- // external refcount. Figure out a better scheme post-promise conversion.
2030
- std::atomic<size_t> external_refs_;
2031
- CallFinalization finalization_;
2032
- CallContext call_context_{this};
2033
- // Contexts for various subsystems (security, tracing, ...).
2034
- grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
2035
- grpc_call_stats final_stats_{};
2036
- Slice final_message_;
2037
- grpc_status_code final_status_ = GRPC_STATUS_UNKNOWN;
2038
- grpc_completion_queue* cq_;
2039
- };
2040
-
2041
- class PromiseBasedCall : public BasicPromiseBasedCall {
2042
- public:
2043
- PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
2044
- const grpc_call_create_args& args);
2045
-
2046
- bool Completed() final { return finished_.IsSet(); }
2047
-
2048
- bool failed_before_recv_message() const final {
2049
- return failed_before_recv_message_.load(std::memory_order_relaxed);
2050
- }
2051
-
2052
- using Call::arena;
2053
-
2054
- protected:
2055
- class ScopedContext : public BasicPromiseBasedCall::ScopedContext,
2056
- public BatchBuilder,
2057
- public promise_detail::Context<BatchBuilder> {
2058
- public:
2059
- explicit ScopedContext(PromiseBasedCall* call)
2060
- : BasicPromiseBasedCall::ScopedContext(call),
2061
- BatchBuilder(&call->batch_payload_),
2062
- promise_detail::Context<BatchBuilder>(this) {}
2063
- };
2064
-
2065
- class Completion {
2066
- public:
2067
- Completion() : index_(kNullIndex) {}
2068
- ~Completion() { CHECK(index_ == kNullIndex); }
2069
- explicit Completion(uint8_t index) : index_(index) {}
2070
- Completion(const Completion& other) = delete;
2071
- Completion& operator=(const Completion& other) = delete;
2072
- Completion(Completion&& other) noexcept : index_(other.index_) {
2073
- other.index_ = kNullIndex;
2074
- }
2075
- Completion& operator=(Completion&& other) noexcept {
2076
- CHECK(index_ == kNullIndex);
2077
- index_ = other.index_;
2078
- other.index_ = kNullIndex;
2079
- return *this;
2080
- }
2081
-
2082
- uint8_t index() const { return index_; }
2083
- uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); }
2084
- bool has_value() const { return index_ != kNullIndex; }
2085
-
2086
- private:
2087
- enum : uint8_t { kNullIndex = 0xff };
2088
- uint8_t index_;
2089
- };
2090
-
2091
- // Enumerates why a Completion is still pending
2092
- enum class PendingOp {
2093
- // We're in the midst of starting a batch of operations
2094
- kStartingBatch = 0,
2095
- // The following correspond with the batch operations from above
2096
- kSendInitialMetadata,
2097
- kReceiveInitialMetadata,
2098
- kReceiveStatusOnClient,
2099
- kReceiveCloseOnServer = kReceiveStatusOnClient,
2100
- kSendMessage,
2101
- kReceiveMessage,
2102
- kSendStatusFromServer,
2103
- kSendCloseFromClient = kSendStatusFromServer,
2104
- };
2105
-
2106
- bool RunParty() override {
2107
- ScopedContext ctx(this);
2108
- return Party::RunParty();
2109
- }
2110
-
2111
- const char* PendingOpString(PendingOp reason) const {
2112
- switch (reason) {
2113
- case PendingOp::kStartingBatch:
2114
- return "StartingBatch";
2115
- case PendingOp::kSendInitialMetadata:
2116
- return "SendInitialMetadata";
2117
- case PendingOp::kReceiveInitialMetadata:
2118
- return "ReceiveInitialMetadata";
2119
- case PendingOp::kReceiveStatusOnClient:
2120
- return is_client() ? "ReceiveStatusOnClient" : "ReceiveCloseOnServer";
2121
- case PendingOp::kSendMessage:
2122
- return "SendMessage";
2123
- case PendingOp::kReceiveMessage:
2124
- return "ReceiveMessage";
2125
- case PendingOp::kSendStatusFromServer:
2126
- return is_client() ? "SendCloseFromClient" : "SendStatusFromServer";
2127
- }
2128
- return "Unknown";
2129
- }
2130
-
2131
- static constexpr uint32_t PendingOpBit(PendingOp reason) {
2132
- return 1 << static_cast<int>(reason);
2133
- }
2134
-
2135
- // Begin work on a completion, recording the tag/closure to notify.
2136
- // Use the op selected in \a ops to determine the index to allocate into.
2137
- // Starts the "StartingBatch" PendingOp immediately.
2138
- // Assumes at least one operation in \a ops.
2139
- Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops);
2140
- // Add one pending op to the completion, and return it.
2141
- Completion AddOpToCompletion(const Completion& completion, PendingOp reason);
2142
- // Stringify a completion
2143
- std::string CompletionString(const Completion& completion) const {
2144
- return completion.has_value()
2145
- ? completion_info_[completion.index()].pending.ToString(this)
2146
- : "no-completion";
2147
- }
2148
- // Finish one op on the completion. Must have been previously been added.
2149
- // The completion as a whole finishes when all pending ops finish.
2150
- void FinishOpOnCompletion(Completion* completion, PendingOp reason);
2151
- // Mark the completion as failed. Does not finish it.
2152
- void FailCompletion(const Completion& completion,
2153
- SourceLocation source_location = {});
2154
- // Mark the completion as infallible. Overrides FailCompletion to report
2155
- // success always.
2156
- void ForceCompletionSuccess(const Completion& completion);
2157
-
2158
- std::string PresentAndCompletionText(const char* caption, bool has,
2159
- const Completion& completion) const {
2160
- if (has) {
2161
- if (completion.has_value()) {
2162
- return absl::StrCat(caption, ":", CompletionString(completion), " ");
2163
- } else {
2164
- return absl::StrCat(caption,
2165
- ":!!BUG:operation is present, no completion!! ");
2166
- }
2167
- } else {
2168
- if (!completion.has_value()) {
2169
- return "";
2170
- } else {
2171
- return absl::StrCat(caption, ":no-op:", CompletionString(completion),
2172
- " ");
2173
- }
2174
- }
2175
- }
2176
-
2177
- // Spawn a job that will first do FirstPromise then receive a message
2178
- template <typename FirstPromise>
2179
- void StartRecvMessage(const grpc_op& op, const Completion& completion,
2180
- FirstPromise first,
2181
- PipeReceiver<MessageHandle>* receiver,
2182
- bool cancel_on_error, Party::BulkSpawner& spawner);
2183
- void StartSendMessage(const grpc_op& op, const Completion& completion,
2184
- PipeSender<MessageHandle>* sender,
2185
- Party::BulkSpawner& spawner);
2186
-
2187
- void set_completed() { finished_.Set(); }
2188
-
2189
- // Returns a promise that resolves to Empty whenever the call is completed.
2190
- auto finished() { return finished_.Wait(); }
2191
-
2192
- // Returns a promise that resolves to Empty whenever there is no outstanding
2193
- // send operation
2194
- auto WaitForSendingStarted() {
2195
- return [this]() -> Poll<Empty> {
2196
- int n = sends_queued_.load(std::memory_order_relaxed);
2197
- if (grpc_call_trace.enabled()) {
2198
- gpr_log(GPR_DEBUG, "%s[call] WaitForSendingStarted n=%d",
2199
- DebugTag().c_str(), n);
2200
- }
2201
- if (n != 0) return waiting_for_queued_sends_.pending();
2202
- return Empty{};
2203
- };
2204
- }
2205
-
2206
- // Mark that a send has been queued - blocks sending trailing metadata.
2207
- void QueueSend() {
2208
- if (grpc_call_trace.enabled()) {
2209
- gpr_log(GPR_DEBUG, "%s[call] QueueSend", DebugTag().c_str());
2210
- }
2211
- sends_queued_.fetch_add(1, std::memory_order_relaxed);
2212
- }
2213
- // Mark that a send has been dequeued - allows sending trailing metadata once
2214
- // zero sends are queued.
2215
- void EnactSend() {
2216
- if (grpc_call_trace.enabled()) {
2217
- gpr_log(GPR_DEBUG, "%s[call] EnactSend", DebugTag().c_str());
2218
- }
2219
- if (1 == sends_queued_.fetch_sub(1, std::memory_order_relaxed)) {
2220
- waiting_for_queued_sends_.Wake();
2221
- }
2222
- }
2223
-
2224
- void set_failed_before_recv_message() {
2225
- failed_before_recv_message_.store(true, std::memory_order_relaxed);
2226
- }
2227
-
2228
- private:
2229
- union CompletionInfo {
2230
- static constexpr uint32_t kOpFailed = 0x8000'0000u;
2231
- static constexpr uint32_t kOpForceSuccess = 0x4000'0000u;
2232
- CompletionInfo() {}
2233
- enum CompletionState {
2234
- kPending,
2235
- kSuccess,
2236
- kFailure,
2237
- };
2238
- struct Pending {
2239
- // Bitmask of PendingOps at the bottom, and kOpFailed, kOpForceSuccess at
2240
- // the top.
2241
- std::atomic<uint32_t> state;
2242
- bool is_closure;
2243
- // True if this completion was for a recv_message op.
2244
- // In that case if the completion as a whole fails we need to cleanup the
2245
- // returned message.
2246
- bool is_recv_message;
2247
- void* tag;
2248
-
2249
- void Start(bool is_closure, void* tag) {
2250
- this->is_closure = is_closure;
2251
- this->is_recv_message = false;
2252
- this->tag = tag;
2253
- state.store(PendingOpBit(PendingOp::kStartingBatch),
2254
- std::memory_order_release);
2255
- }
2256
-
2257
- void AddPendingBit(PendingOp reason) {
2258
- if (reason == PendingOp::kReceiveMessage) is_recv_message = true;
2259
- auto prev =
2260
- state.fetch_or(PendingOpBit(reason), std::memory_order_relaxed);
2261
- CHECK_EQ((prev & PendingOpBit(reason)), 0u);
2262
- }
2263
-
2264
- CompletionState RemovePendingBit(PendingOp reason) {
2265
- const uint32_t mask = ~PendingOpBit(reason);
2266
- auto prev = state.fetch_and(mask, std::memory_order_acq_rel);
2267
- CHECK_NE((prev & PendingOpBit(reason)), 0u);
2268
- switch (prev & mask) {
2269
- case kOpFailed:
2270
- return kFailure;
2271
- case kOpFailed | kOpForceSuccess:
2272
- case kOpForceSuccess:
2273
- case 0:
2274
- return kSuccess;
2275
- default:
2276
- return kPending;
2277
- }
2278
- }
2279
-
2280
- void MarkFailed() {
2281
- state.fetch_or(kOpFailed, std::memory_order_relaxed);
2282
- }
2283
-
2284
- void MarkForceSuccess() {
2285
- state.fetch_or(kOpForceSuccess, std::memory_order_relaxed);
2286
- }
2287
-
2288
- std::string ToString(const PromiseBasedCall* call) const {
2289
- auto state = this->state.load(std::memory_order_relaxed);
2290
- std::vector<absl::string_view> pending_ops;
2291
- for (size_t i = 0; i < 24; i++) {
2292
- if (state & (1u << i)) {
2293
- pending_ops.push_back(
2294
- call->PendingOpString(static_cast<PendingOp>(i)));
2295
- }
2296
- }
2297
- return absl::StrFormat("{%s}%s:tag=%p", absl::StrJoin(pending_ops, ","),
2298
- (state & kOpForceSuccess) ? ":force-success"
2299
- : (state & kOpFailed) ? ":failed"
2300
- : ":success",
2301
- tag);
2302
- }
2303
- } pending;
2304
- grpc_cq_completion completion;
2305
- };
2306
-
2307
- CompletionInfo completion_info_[6];
2308
- ExternallyObservableLatch<void> finished_;
2309
- // Non-zero with an outstanding GRPC_OP_SEND_INITIAL_METADATA or
2310
- // GRPC_OP_SEND_MESSAGE (one count each), and 0 once those payloads have been
2311
- // pushed onto the outgoing pipe.
2312
- std::atomic<uint8_t> sends_queued_{0};
2313
- std::atomic<bool> failed_before_recv_message_{false};
2314
- // Waiter for when sends_queued_ becomes 0.
2315
- IntraActivityWaiter waiting_for_queued_sends_;
2316
- grpc_byte_buffer** recv_message_ = nullptr;
2317
- grpc_transport_stream_op_batch_payload batch_payload_{context()};
2318
- };
2319
-
2320
- template <typename T>
2321
- grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args,
2322
- grpc_call** out_call) {
2323
- Channel* channel = args->channel.get();
2324
-
2325
- auto* arena = channel->CreateArena();
2326
- PromiseBasedCall* call = arena->New<T>(arena, args);
2327
- *out_call = call->c_ptr();
2328
- DCHECK(Call::FromC(*out_call) == call);
2329
- return absl::OkStatus();
2330
- }
2331
-
2332
- PromiseBasedCall::PromiseBasedCall(Arena* arena, uint32_t initial_external_refs,
2333
- const grpc_call_create_args& args)
2334
- : BasicPromiseBasedCall(arena, initial_external_refs,
2335
- initial_external_refs != 0 ? 1 : 0, args) {}
2336
-
2337
- static void CToMetadata(grpc_metadata* metadata, size_t count,
2338
- grpc_metadata_batch* b) {
2339
- for (size_t i = 0; i < count; i++) {
2340
- grpc_metadata* md = &metadata[i];
2341
- auto key = StringViewFromSlice(md->key);
2342
- // Filter "content-length metadata"
2343
- if (key == "content-length") continue;
2344
- b->Append(key, Slice(CSliceRef(md->value)),
2345
- [md](absl::string_view error, const Slice& value) {
2346
- gpr_log(GPR_DEBUG, "Append error: %s",
2347
- absl::StrCat("key=", StringViewFromSlice(md->key),
2348
- " error=", error,
2349
- " value=", value.as_string_view())
2350
- .c_str());
2351
- });
2352
- }
2353
- }
2354
-
2355
- PromiseBasedCall::Completion PromiseBasedCall::StartCompletion(
2356
- void* tag, bool is_closure, const grpc_op* ops) {
2357
- Completion c(BatchSlotForOp(ops[0].op));
2358
- if (!is_closure) {
2359
- grpc_cq_begin_op(cq(), tag);
2360
- }
2361
- completion_info_[c.index()].pending.Start(is_closure, tag);
2362
- if (grpc_call_trace.enabled()) {
2363
- gpr_log(GPR_INFO, "%s[call] StartCompletion %s", DebugTag().c_str(),
2364
- CompletionString(c).c_str());
2365
- }
2366
- return c;
2367
- }
2368
-
2369
- PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion(
2370
- const Completion& completion, PendingOp reason) {
2371
- if (grpc_call_trace.enabled()) {
2372
- gpr_log(GPR_INFO, "%s[call] AddOpToCompletion %s %s", DebugTag().c_str(),
2373
- CompletionString(completion).c_str(), PendingOpString(reason));
2374
- }
2375
- CHECK(completion.has_value());
2376
- completion_info_[completion.index()].pending.AddPendingBit(reason);
2377
- return Completion(completion.index());
2378
- }
2379
-
2380
- void PromiseBasedCall::FailCompletion(const Completion& completion,
2381
- SourceLocation location) {
2382
- if (grpc_call_trace.enabled()) {
2383
- gpr_log(location.file(), location.line(), GPR_LOG_SEVERITY_ERROR,
2384
- "%s[call] FailCompletion %s", DebugTag().c_str(),
2385
- CompletionString(completion).c_str());
2386
- }
2387
- completion_info_[completion.index()].pending.MarkFailed();
2388
- }
2389
-
2390
- void PromiseBasedCall::ForceCompletionSuccess(const Completion& completion) {
2391
- completion_info_[completion.index()].pending.MarkForceSuccess();
2392
- }
2393
-
2394
- void PromiseBasedCall::FinishOpOnCompletion(Completion* completion,
2395
- PendingOp reason) {
2396
- if (grpc_call_trace.enabled()) {
2397
- gpr_log(GPR_INFO, "%s[call] FinishOpOnCompletion completion:%s finish:%s",
2398
- DebugTag().c_str(), CompletionString(*completion).c_str(),
2399
- PendingOpString(reason));
2400
- }
2401
- const uint8_t i = completion->TakeIndex();
2402
- CHECK(i < GPR_ARRAY_SIZE(completion_info_));
2403
- CompletionInfo::Pending& pending = completion_info_[i].pending;
2404
- bool success;
2405
- switch (pending.RemovePendingBit(reason)) {
2406
- case CompletionInfo::kPending:
2407
- return; // Early out
2408
- case CompletionInfo::kSuccess:
2409
- success = true;
2410
- break;
2411
- case CompletionInfo::kFailure:
2412
- success = false;
2413
- break;
2414
- }
2415
- if (pending.is_recv_message && !success && *recv_message_ != nullptr) {
2416
- grpc_byte_buffer_destroy(*recv_message_);
2417
- *recv_message_ = nullptr;
2418
- }
2419
- auto error = success ? absl::OkStatus() : absl::CancelledError();
2420
- if (pending.is_closure) {
2421
- ExecCtx::Run(DEBUG_LOCATION, static_cast<grpc_closure*>(pending.tag),
2422
- error);
2423
- } else {
2424
- InternalRef("cq_end_op");
2425
- grpc_cq_end_op(
2426
- cq(), pending.tag, error,
2427
- [](void* p, grpc_cq_completion*) {
2428
- static_cast<PromiseBasedCall*>(p)->InternalUnref("cq_end_op");
2429
- },
2430
- this, &completion_info_[i].completion);
2431
- }
2432
- }
2433
-
2434
- void PromiseBasedCall::StartSendMessage(const grpc_op& op,
2435
- const Completion& completion,
2436
- PipeSender<MessageHandle>* sender,
2437
- Party::BulkSpawner& spawner) {
2438
- QueueSend();
2439
- SliceBuffer send;
2440
- grpc_slice_buffer_swap(
2441
- &op.data.send_message.send_message->data.raw.slice_buffer,
2442
- send.c_slice_buffer());
2443
- auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
2444
- spawner.Spawn(
2445
- "call_send_message",
2446
- [this, sender, msg = std::move(msg)]() mutable {
2447
- EnactSend();
2448
- return sender->Push(std::move(msg));
2449
- },
2450
- [this, completion = AddOpToCompletion(
2451
- completion, PendingOp::kSendMessage)](bool result) mutable {
2452
- if (grpc_call_trace.enabled()) {
2453
- gpr_log(GPR_DEBUG, "%sSendMessage completes %s", DebugTag().c_str(),
2454
- result ? "successfully" : "with failure");
2455
- }
2456
- if (!result) FailCompletion(completion);
2457
- FinishOpOnCompletion(&completion, PendingOp::kSendMessage);
2458
- });
2459
- }
2460
-
2461
- template <typename FirstPromiseFactory>
2462
- void PromiseBasedCall::StartRecvMessage(
2463
- const grpc_op& op, const Completion& completion,
2464
- FirstPromiseFactory first_promise_factory,
2465
- PipeReceiver<MessageHandle>* receiver, bool cancel_on_error,
2466
- Party::BulkSpawner& spawner) {
2467
- if (grpc_call_trace.enabled()) {
2468
- gpr_log(GPR_INFO, "%s[call] Start RecvMessage: %s", DebugTag().c_str(),
2469
- CompletionString(completion).c_str());
2470
- }
2471
- recv_message_ = op.data.recv_message.recv_message;
2472
- spawner.Spawn(
2473
- "call_recv_message",
2474
- [first_promise_factory = std::move(first_promise_factory), receiver]() {
2475
- return Seq(first_promise_factory(), receiver->Next());
2476
- },
2477
- [this, cancel_on_error,
2478
- completion = AddOpToCompletion(completion, PendingOp::kReceiveMessage)](
2479
- NextResult<MessageHandle> result) mutable {
2480
- if (result.has_value()) {
2481
- MessageHandle& message = *result;
2482
- NoteLastMessageFlags(message->flags());
2483
- if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
2484
- (incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
2485
- *recv_message_ = grpc_raw_compressed_byte_buffer_create(
2486
- nullptr, 0, incoming_compression_algorithm());
2487
- } else {
2488
- *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
2489
- }
2490
- grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
2491
- &(*recv_message_)->data.raw.slice_buffer);
2492
- if (grpc_call_trace.enabled()) {
2493
- gpr_log(GPR_INFO,
2494
- "%s[call] RecvMessage: outstanding_recv "
2495
- "finishes: received %" PRIdPTR " byte message",
2496
- DebugTag().c_str(),
2497
- (*recv_message_)->data.raw.slice_buffer.length);
2498
- }
2499
- } else if (result.cancelled()) {
2500
- if (grpc_call_trace.enabled()) {
2501
- gpr_log(GPR_INFO,
2502
- "%s[call] RecvMessage: outstanding_recv "
2503
- "finishes: received end-of-stream with error",
2504
- DebugTag().c_str());
2505
- }
2506
- set_failed_before_recv_message();
2507
- FailCompletion(completion);
2508
- if (cancel_on_error) CancelWithError(absl::CancelledError());
2509
- *recv_message_ = nullptr;
2510
- } else {
2511
- if (grpc_call_trace.enabled()) {
2512
- gpr_log(GPR_INFO,
2513
- "%s[call] RecvMessage: outstanding_recv "
2514
- "finishes: received end-of-stream",
2515
- DebugTag().c_str());
2516
- }
2517
- *recv_message_ = nullptr;
2518
- }
2519
- FinishOpOnCompletion(&completion, PendingOp::kReceiveMessage);
2520
- });
2521
- }
2522
-
2523
- ///////////////////////////////////////////////////////////////////////////////
2524
- // CallContext
2525
-
2526
- void CallContext::RunInContext(absl::AnyInvocable<void()> fn) {
2527
- call_->RunInContext(std::move(fn));
2528
- }
2529
-
2530
- void CallContext::IncrementRefCount(const char* reason) {
2531
- call_->InternalRef(reason);
2532
- }
2533
-
2534
- void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); }
2535
-
2536
- ServerCallContext* CallContext::server_call_context() {
2537
- return call_->server_call_context();
2538
- }
2539
-
2540
- RefCountedPtr<CallSpineInterface> CallContext::MakeCallSpine(
2541
- CallArgs call_args) {
2542
- return call_->MakeCallSpine(std::move(call_args));
2543
- }
2544
-
2545
- grpc_call* CallContext::c_call() { return call_->c_ptr(); }
2546
-
2547
- ///////////////////////////////////////////////////////////////////////////////
2548
- // PublishMetadataArray
2549
-
2550
- namespace {
2551
- void PublishMetadataArray(grpc_metadata_batch* md, grpc_metadata_array* array,
2552
- bool is_client) {
2553
- const auto md_count = md->count();
2554
- if (md_count > array->capacity) {
2555
- array->capacity =
2556
- std::max(array->capacity + md->count(), array->capacity * 3 / 2);
2557
- array->metadata = static_cast<grpc_metadata*>(
2558
- gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity));
2559
- }
2560
- PublishToAppEncoder encoder(array, md, is_client);
2561
- md->Encode(&encoder);
2562
- }
2563
- } // namespace
2564
-
2565
- ///////////////////////////////////////////////////////////////////////////////
2566
- // ClientPromiseBasedCall
2567
-
2568
- #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
2569
- class ClientPromiseBasedCall final : public PromiseBasedCall {
2570
- public:
2571
- ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args)
2572
- : PromiseBasedCall(arena, 1, *args),
2573
- polling_entity_(
2574
- args->cq != nullptr
2575
- ? grpc_polling_entity_create_from_pollset(
2576
- grpc_cq_pollset(args->cq))
2577
- : (args->pollset_set_alternative != nullptr
2578
- ? grpc_polling_entity_create_from_pollset_set(
2579
- args->pollset_set_alternative)
2580
- : grpc_polling_entity{})) {
2581
- global_stats().IncrementClientCallsCreated();
2582
- if (args->cq != nullptr) {
2583
- CHECK(args->pollset_set_alternative == nullptr)
2584
- << "Only one of 'cq' and 'pollset_set_alternative' should be "
2585
- "non-nullptr.";
2586
- }
2587
- ScopedContext context(this);
2588
- args->channel->channel_stack()->stats_plugin_group->AddClientCallTracers(
2589
- *args->path, args->registered_method, this->context());
2590
- send_initial_metadata_ = Arena::MakePooled<ClientMetadata>();
2591
- send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path));
2592
- if (args->authority.has_value()) {
2593
- send_initial_metadata_->Set(HttpAuthorityMetadata(),
2594
- std::move(*args->authority));
2595
- }
2596
- send_initial_metadata_->Set(GrpcRegisteredMethod(),
2597
- reinterpret_cast<void*>(static_cast<uintptr_t>(
2598
- args->registered_method)));
2599
- if (auto* channelz_channel = channel()->channelz_node()) {
2600
- channelz_channel->RecordCallStarted();
2601
- }
2602
- if (args->send_deadline != Timestamp::InfFuture()) {
2603
- UpdateDeadline(args->send_deadline);
2604
- }
2605
- Call* parent = Call::FromC(args->parent);
2606
- if (parent != nullptr) {
2607
- auto parent_status = InitParent(parent, args->propagation_mask);
2608
- if (!parent_status.ok()) {
2609
- CancelWithError(std::move(parent_status));
2610
- }
2611
- PublishToParent(parent);
2612
- }
2613
- }
2614
-
2615
- void OrphanCall() override { MaybeUnpublishFromParent(); }
2616
-
2617
- ~ClientPromiseBasedCall() override {
2618
- ScopedContext context(this);
2619
- send_initial_metadata_.reset();
2620
- // Need to destroy the pipes under the ScopedContext above, so we
2621
- // move them out here and then allow the destructors to run at
2622
- // end of scope, but before context.
2623
- auto c2s = std::move(client_to_server_messages_);
2624
- auto s2c = std::move(server_to_client_messages_);
2625
- auto sim = std::move(server_initial_metadata_);
2626
- }
2627
-
2628
- void CancelWithError(absl::Status error) override {
2629
- if (cancel_with_error_called_.exchange(true, std::memory_order_relaxed)) {
2630
- return;
2631
- }
2632
- if (!started_.exchange(true, std::memory_order_relaxed)) {
2633
- // Initial metadata not sent yet, so we can just fail the call.
2634
- Spawn(
2635
- "cancel_before_initial_metadata",
2636
- [error = std::move(error), this]() {
2637
- server_to_client_messages_.sender.Close();
2638
- auto md = ServerMetadataFromStatus(error);
2639
- md->Set(GrpcCallWasCancelled(), true);
2640
- Finish(std::move(md));
2641
- return Empty{};
2642
- },
2643
- [](Empty) {});
2644
- } else {
2645
- Spawn(
2646
- "cancel_with_error",
2647
- [error = std::move(error), this]() {
2648
- if (!cancel_error_.is_set()) {
2649
- auto md = ServerMetadataFromStatus(error);
2650
- md->Set(GrpcCallWasCancelled(), true);
2651
- cancel_error_.Set(std::move(md));
2652
- }
2653
- return Empty{};
2654
- },
2655
- [](Empty) {});
2656
- }
2657
- }
2658
- absl::string_view GetServerAuthority() const override { abort(); }
2659
- bool is_trailers_only() const override { return is_trailers_only_; }
2660
-
2661
- grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
2662
- bool is_notify_tag_closure) override;
2663
-
2664
- std::string DebugTag() const override {
2665
- return absl::StrFormat("CLIENT_CALL[%p]: ", this);
2666
- }
2667
-
2668
- RefCountedPtr<CallSpineInterface> MakeCallSpine(CallArgs call_args) final {
2669
- class WrappingCallSpine final : public PipeBasedCallSpine {
2670
- public:
2671
- WrappingCallSpine(ClientPromiseBasedCall* call,
2672
- ClientMetadataHandle metadata)
2673
- : call_(call) {
2674
- call_->InternalRef("call-spine");
2675
- SpawnInfallible(
2676
- "send_client_initial_metadata",
2677
- [self = Ref(), metadata = std::move(metadata)]() mutable {
2678
- return Map(self->client_initial_metadata_.sender.Push(
2679
- std::move(metadata)),
2680
- [self](bool) { return Empty{}; });
2681
- });
2682
- }
2683
-
2684
- ~WrappingCallSpine() override {
2685
- {
2686
- ScopedContext context(call_);
2687
- // Move these out and destroy before the internal unref.
2688
- auto client_initial_metadata = std::move(client_initial_metadata_);
2689
- auto server_trailing_metadata = std::move(server_trailing_metadata_);
2690
- }
2691
- call_->InternalUnref("call-spine");
2692
- }
2693
-
2694
- Pipe<ClientMetadataHandle>& client_initial_metadata() override {
2695
- return client_initial_metadata_;
2696
- }
2697
-
2698
- Pipe<MessageHandle>& client_to_server_messages() override {
2699
- return call_->client_to_server_messages_;
2700
- }
2701
-
2702
- Pipe<ServerMetadataHandle>& server_initial_metadata() override {
2703
- return call_->server_initial_metadata_;
2704
- }
2705
-
2706
- Pipe<MessageHandle>& server_to_client_messages() override {
2707
- return call_->server_to_client_messages_;
2708
- }
2709
-
2710
- Latch<ServerMetadataHandle>& cancel_latch() override {
2711
- return cancel_error_;
2712
- }
2713
-
2714
- Latch<bool>& was_cancelled_latch() override {
2715
- return was_cancelled_latch_;
2716
- }
2717
-
2718
- Party& party() override { return *call_; }
2719
- Arena* arena() override { return call_->arena(); }
2720
-
2721
- void IncrementRefCount() override { refs_.Ref(); }
2722
- void Unref() override {
2723
- if (refs_.Unref()) delete this;
2724
- }
2725
- RefCountedPtr<WrappingCallSpine> Ref() {
2726
- IncrementRefCount();
2727
- return RefCountedPtr<WrappingCallSpine>(this);
2728
- }
2729
-
2730
- ClientMetadata& UnprocessedClientInitialMetadata() override {
2731
- Crash("not for v2");
2732
- }
2733
-
2734
- void V2HackToStartCallWithoutACallFilterStack() override {}
2735
-
2736
- private:
2737
- RefCount refs_;
2738
- ClientPromiseBasedCall* const call_;
2739
- std::atomic<bool> sent_trailing_metadata_{false};
2740
- Pipe<ClientMetadataHandle> client_initial_metadata_{call_->arena()};
2741
- Pipe<ServerMetadataHandle> server_trailing_metadata_{call_->arena()};
2742
- Latch<ServerMetadataHandle> cancel_error_;
2743
- Latch<bool> was_cancelled_latch_;
2744
- };
2745
- CHECK(call_args.server_initial_metadata ==
2746
- &server_initial_metadata_.sender);
2747
- CHECK(call_args.client_to_server_messages ==
2748
- &client_to_server_messages_.receiver);
2749
- CHECK(call_args.server_to_client_messages ==
2750
- &server_to_client_messages_.sender);
2751
- call_args.client_initial_metadata_outstanding.Complete(true);
2752
- return MakeRefCounted<WrappingCallSpine>(
2753
- this, std::move(call_args.client_initial_metadata));
2754
- }
2755
-
2756
- private:
2757
- // Finish the call with the given status/trailing metadata.
2758
- void Finish(ServerMetadataHandle trailing_metadata);
2759
- // Validate that a set of ops is valid for a client call.
2760
- grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const;
2761
- // Commit a valid batch of operations to be executed.
2762
- void CommitBatch(const grpc_op* ops, size_t nops,
2763
- const Completion& completion);
2764
- // Start the underlying promise.
2765
- void StartPromise(ClientMetadataHandle client_initial_metadata,
2766
- const Completion& completion, Party::BulkSpawner& spawner);
2767
- // Start receiving initial metadata
2768
- void StartRecvInitialMetadata(grpc_metadata_array* array,
2769
- const Completion& completion,
2770
- Party::BulkSpawner& spawner);
2771
- void StartRecvStatusOnClient(
2772
- const Completion& completion,
2773
- grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
2774
- Party::BulkSpawner& spawner);
2775
- // Publish status out to the application.
2776
- void PublishStatus(
2777
- grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
2778
- ServerMetadataHandle trailing_metadata);
2779
- // Publish server initial metadata out to the application.
2780
- void PublishInitialMetadata(ServerMetadata* metadata);
2781
-
2782
- ClientMetadataHandle send_initial_metadata_;
2783
- Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
2784
- Latch<ServerMetadataHandle> server_trailing_metadata_;
2785
- Latch<ServerMetadataHandle> cancel_error_;
2786
- Latch<grpc_polling_entity> polling_entity_;
2787
- Pipe<MessageHandle> client_to_server_messages_{arena()};
2788
- Pipe<MessageHandle> server_to_client_messages_{arena()};
2789
- bool is_trailers_only_ = false;
2790
- bool scheduled_receive_status_ = false;
2791
- bool scheduled_send_close_ = false;
2792
- // True once the promise for the call is started.
2793
- // This corresponds to sending initial metadata, or cancelling before doing
2794
- // so.
2795
- // In the latter case real world code sometimes does not sent the initial
2796
- // metadata, and so gating based upon that does not work out.
2797
- std::atomic<bool> started_{false};
2798
- // True after the first CancelWithError call - prevents spamming cancels from
2799
- // overflowing the party.
2800
- std::atomic<bool> cancel_with_error_called_{false};
2801
- // TODO(ctiller): delete when we remove the filter based API (may require some
2802
- // cleanup in wrapped languages: they depend on this to hold slice refs)
2803
- ServerMetadataHandle recv_initial_metadata_;
2804
- ServerMetadataHandle recv_trailing_metadata_;
2805
- };
2806
-
2807
- void ClientPromiseBasedCall::StartPromise(
2808
- ClientMetadataHandle client_initial_metadata, const Completion& completion,
2809
- Party::BulkSpawner& spawner) {
2810
- auto token = ClientInitialMetadataOutstandingToken::New(arena());
2811
- spawner.Spawn(
2812
- "call_send_initial_metadata", token.Wait(),
2813
- [this,
2814
- completion = AddOpToCompletion(
2815
- completion, PendingOp::kSendInitialMetadata)](bool result) mutable {
2816
- if (!result) FailCompletion(completion);
2817
- FinishOpOnCompletion(&completion, PendingOp::kSendInitialMetadata);
2818
- });
2819
- spawner.Spawn(
2820
- "client_promise",
2821
- [this, client_initial_metadata = std::move(client_initial_metadata),
2822
- token = std::move(token)]() mutable {
2823
- return Race(
2824
- cancel_error_.Wait(),
2825
- Map(channel()->channel_stack()->MakeClientCallPromise(CallArgs{
2826
- std::move(client_initial_metadata), std::move(token),
2827
- &polling_entity_, &server_initial_metadata_.sender,
2828
- &client_to_server_messages_.receiver,
2829
- &server_to_client_messages_.sender}),
2830
- [this](ServerMetadataHandle trailing_metadata) {
2831
- // If we're cancelled the transport doesn't get to return
2832
- // stats.
2833
- AcceptTransportStatsFromContext();
2834
- return trailing_metadata;
2835
- }));
2836
- },
2837
- [this](ServerMetadataHandle trailing_metadata) {
2838
- Finish(std::move(trailing_metadata));
2839
- });
2840
- }
2841
-
2842
- grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops,
2843
- size_t nops) const {
2844
- BitSet<8> got_ops;
2845
- for (size_t op_idx = 0; op_idx < nops; op_idx++) {
2846
- const grpc_op& op = ops[op_idx];
2847
- switch (op.op) {
2848
- case GRPC_OP_SEND_INITIAL_METADATA:
2849
- if (!AreInitialMetadataFlagsValid(op.flags)) {
2850
- return GRPC_CALL_ERROR_INVALID_FLAGS;
2851
- }
2852
- if (!ValidateMetadata(op.data.send_initial_metadata.count,
2853
- op.data.send_initial_metadata.metadata)) {
2854
- return GRPC_CALL_ERROR_INVALID_METADATA;
2855
- }
2856
- break;
2857
- case GRPC_OP_SEND_MESSAGE:
2858
- if (!AreWriteFlagsValid(op.flags)) {
2859
- return GRPC_CALL_ERROR_INVALID_FLAGS;
2860
- }
2861
- break;
2862
- case GRPC_OP_RECV_INITIAL_METADATA:
2863
- case GRPC_OP_RECV_MESSAGE:
2864
- if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
2865
- break;
2866
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
2867
- if (scheduled_send_close_) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
2868
- if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
2869
- break;
2870
- case GRPC_OP_RECV_STATUS_ON_CLIENT:
2871
- if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
2872
- if (scheduled_receive_status_) {
2873
- return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
2874
- }
2875
- break;
2876
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
2877
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
2878
- return GRPC_CALL_ERROR_NOT_ON_CLIENT;
2879
- }
2880
- if (got_ops.is_set(op.op)) {
2881
- return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
2882
- }
2883
- got_ops.set(op.op);
2884
- }
2885
- return GRPC_CALL_OK;
2886
- }
2887
-
2888
- void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
2889
- const Completion& completion) {
2890
- Party::BulkSpawner spawner(this);
2891
- for (size_t op_idx = 0; op_idx < nops; op_idx++) {
2892
- const grpc_op& op = ops[op_idx];
2893
- switch (op.op) {
2894
- case GRPC_OP_SEND_INITIAL_METADATA: {
2895
- if (started_.exchange(true, std::memory_order_relaxed)) break;
2896
- CToMetadata(op.data.send_initial_metadata.metadata,
2897
- op.data.send_initial_metadata.count,
2898
- send_initial_metadata_.get());
2899
- PrepareOutgoingInitialMetadata(op, *send_initial_metadata_);
2900
- if (send_deadline() != Timestamp::InfFuture()) {
2901
- send_initial_metadata_->Set(GrpcTimeoutMetadata(), send_deadline());
2902
- }
2903
- send_initial_metadata_->Set(
2904
- WaitForReady(),
2905
- WaitForReady::ValueType{
2906
- (op.flags & GRPC_INITIAL_METADATA_WAIT_FOR_READY) != 0,
2907
- (op.flags &
2908
- GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET) != 0});
2909
- StartPromise(std::move(send_initial_metadata_), completion, spawner);
2910
- } break;
2911
- case GRPC_OP_RECV_INITIAL_METADATA: {
2912
- StartRecvInitialMetadata(
2913
- op.data.recv_initial_metadata.recv_initial_metadata, completion,
2914
- spawner);
2915
- } break;
2916
- case GRPC_OP_RECV_STATUS_ON_CLIENT: {
2917
- scheduled_receive_status_ = true;
2918
- StartRecvStatusOnClient(completion, op.data.recv_status_on_client,
2919
- spawner);
2920
- } break;
2921
- case GRPC_OP_SEND_MESSAGE:
2922
- StartSendMessage(op, completion, &client_to_server_messages_.sender,
2923
- spawner);
2924
- break;
2925
- case GRPC_OP_RECV_MESSAGE:
2926
- StartRecvMessage(
2927
- op, completion,
2928
- [this]() {
2929
- return Race(server_initial_metadata_.receiver.AwaitClosed(),
2930
- server_to_client_messages_.receiver.AwaitClosed());
2931
- },
2932
- &server_to_client_messages_.receiver, false, spawner);
2933
- break;
2934
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
2935
- scheduled_send_close_ = true;
2936
- spawner.Spawn(
2937
- "send_close_from_client",
2938
- [this]() {
2939
- client_to_server_messages_.sender.Close();
2940
- return Empty{};
2941
- },
2942
- [this,
2943
- completion = AddOpToCompletion(
2944
- completion, PendingOp::kSendCloseFromClient)](Empty) mutable {
2945
- FinishOpOnCompletion(&completion,
2946
- PendingOp::kSendCloseFromClient);
2947
- });
2948
- break;
2949
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
2950
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
2951
- abort(); // unreachable
2952
- }
2953
- }
2954
- }
2955
-
2956
- grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops,
2957
- size_t nops,
2958
- void* notify_tag,
2959
- bool is_notify_tag_closure) {
2960
- if (nops == 0) {
2961
- EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
2962
- return GRPC_CALL_OK;
2963
- }
2964
- const grpc_call_error validation_result = ValidateBatch(ops, nops);
2965
- if (validation_result != GRPC_CALL_OK) {
2966
- return validation_result;
2967
- }
2968
- Completion completion =
2969
- StartCompletion(notify_tag, is_notify_tag_closure, ops);
2970
- CommitBatch(ops, nops, completion);
2971
- FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
2972
- return GRPC_CALL_OK;
2973
- }
2974
-
2975
- void ClientPromiseBasedCall::StartRecvInitialMetadata(
2976
- grpc_metadata_array* array, const Completion& completion,
2977
- Party::BulkSpawner& spawner) {
2978
- spawner.Spawn(
2979
- "recv_initial_metadata",
2980
- [this]() {
2981
- return Race(server_initial_metadata_.receiver.Next(),
2982
- Map(finished(), [](Empty) {
2983
- return NextResult<ServerMetadataHandle>(true);
2984
- }));
2985
- },
2986
- [this, array,
2987
- completion =
2988
- AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata)](
2989
- NextResult<ServerMetadataHandle> next_metadata) mutable {
2990
- server_initial_metadata_.sender.Close();
2991
- ServerMetadataHandle metadata;
2992
- if (grpc_call_trace.enabled()) {
2993
- gpr_log(GPR_INFO, "%s[call] RecvTrailingMetadata: %s",
2994
- DebugTag().c_str(),
2995
- next_metadata.has_value()
2996
- ? next_metadata.value()->DebugString().c_str()
2997
- : "null");
2998
- }
2999
- if (next_metadata.has_value()) {
3000
- metadata = std::move(next_metadata.value());
3001
- is_trailers_only_ = metadata->get(GrpcTrailersOnly()).value_or(false);
3002
- } else {
3003
- is_trailers_only_ = true;
3004
- metadata = arena()->MakePooled<ServerMetadata>();
3005
- }
3006
- ProcessIncomingInitialMetadata(*metadata);
3007
- PublishMetadataArray(metadata.get(), array, true);
3008
- recv_initial_metadata_ = std::move(metadata);
3009
- FinishOpOnCompletion(&completion, PendingOp::kReceiveInitialMetadata);
3010
- });
3011
- }
3012
-
3013
- void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) {
3014
- if (grpc_call_trace.enabled()) {
3015
- gpr_log(GPR_INFO, "%s[call] Finish: %s", DebugTag().c_str(),
3016
- trailing_metadata->DebugString().c_str());
3017
- }
3018
- ResetDeadline();
3019
- set_completed();
3020
- client_to_server_messages_.sender.CloseWithError();
3021
- client_to_server_messages_.receiver.CloseWithError();
3022
- if (trailing_metadata->get(GrpcCallWasCancelled()).value_or(false)) {
3023
- server_to_client_messages_.receiver.CloseWithError();
3024
- server_initial_metadata_.receiver.CloseWithError();
3025
- }
3026
- if (auto* channelz_channel = channel()->channelz_node()) {
3027
- if (trailing_metadata->get(GrpcStatusMetadata())
3028
- .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) {
3029
- channelz_channel->RecordCallSucceeded();
3030
- } else {
3031
- channelz_channel->RecordCallFailed();
3032
- }
3033
- }
3034
- server_trailing_metadata_.Set(std::move(trailing_metadata));
3035
- }
3036
-
3037
- namespace {
3038
- std::string MakeErrorString(const ServerMetadata* trailing_metadata) {
3039
- std::string out = absl::StrCat(
3040
- trailing_metadata->get(GrpcStatusFromWire()).value_or(false)
3041
- ? "Error received from peer"
3042
- : "Error generated by client",
3043
- "grpc_status: ",
3044
- grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata())
3045
- .value_or(GRPC_STATUS_UNKNOWN)));
3046
- if (const Slice* message =
3047
- trailing_metadata->get_pointer(GrpcMessageMetadata())) {
3048
- absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view());
3049
- }
3050
- if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) {
3051
- absl::StrAppend(&out, "\nStatus Context:");
3052
- for (const std::string& annotation : *annotations) {
3053
- absl::StrAppend(&out, "\n ", annotation);
3054
- }
3055
- }
3056
- return out;
3057
- }
3058
- } // namespace
3059
-
3060
- void ClientPromiseBasedCall::StartRecvStatusOnClient(
3061
- const Completion& completion,
3062
- grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args,
3063
- Party::BulkSpawner& spawner) {
3064
- ForceCompletionSuccess(completion);
3065
- spawner.Spawn(
3066
- "recv_status_on_client", server_trailing_metadata_.Wait(),
3067
- [this, op_args,
3068
- completion =
3069
- AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient)](
3070
- ServerMetadataHandle trailing_metadata) mutable {
3071
- const grpc_status_code status =
3072
- trailing_metadata->get(GrpcStatusMetadata())
3073
- .value_or(GRPC_STATUS_UNKNOWN);
3074
- *op_args.status = status;
3075
- Slice message_slice;
3076
- if (Slice* message =
3077
- trailing_metadata->get_pointer(GrpcMessageMetadata())) {
3078
- message_slice = message->Ref();
3079
- }
3080
- SetFinalizationStatus(status, message_slice.Ref());
3081
- *op_args.status_details = message_slice.TakeCSlice();
3082
- if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) {
3083
- *op_args.error_string =
3084
- gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str());
3085
- }
3086
- PublishMetadataArray(trailing_metadata.get(), op_args.trailing_metadata,
3087
- true);
3088
- recv_trailing_metadata_ = std::move(trailing_metadata);
3089
- FinishOpOnCompletion(&completion, PendingOp::kReceiveStatusOnClient);
3090
- });
3091
- }
3092
- #endif
3093
-
3094
- ///////////////////////////////////////////////////////////////////////////////
3095
- // ServerPromiseBasedCall
3096
-
3097
- #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
3098
-
3099
- class ServerPromiseBasedCall final : public PromiseBasedCall,
3100
- public ServerCallContext {
3101
- public:
3102
- ServerPromiseBasedCall(Arena* arena, grpc_call_create_args* args);
3103
-
3104
- void OrphanCall() override {}
3105
- void CancelWithError(grpc_error_handle) override;
3106
- grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
3107
- bool is_notify_tag_closure) override;
3108
- bool is_trailers_only() const override {
3109
- Crash("is_trailers_only not implemented for server calls");
3110
- }
3111
- absl::string_view GetServerAuthority() const override {
3112
- const Slice* authority_metadata =
3113
- client_initial_metadata_->get_pointer(HttpAuthorityMetadata());
3114
- if (authority_metadata == nullptr) return "";
3115
- return authority_metadata->as_string_view();
3116
- }
3117
-
3118
- // Polling order for the server promise stack:
3119
- //
3120
- // │ ┌───────────────────────────────────────┐
3121
- // │ │ ServerPromiseBasedCall ├──► Lifetime management
3122
- // │ ├───────────────────────────────────────┤
3123
- // │ │ ConnectedChannel ├─┐
3124
- // │ ├───────────────────────────────────────┤ └► Interactions with the
3125
- // │ │ ... closest to transport filter │ transport - send/recv msgs
3126
- // │ ├───────────────────────────────────────┤ and metadata, call phase
3127
- // │ │ ... │ ordering
3128
- // │ ├───────────────────────────────────────┤
3129
- // │ │ ... closest to app filter │ ┌► Request matching, initial
3130
- // │ ├───────────────────────────────────────┤ │ setup, publishing call to
3131
- // │ │ Server::ChannelData::MakeCallPromise ├─┘ application
3132
- // │ ├───────────────────────────────────────┤
3133
- // │ │ MakeTopOfServerCallPromise ├──► Send trailing metadata
3134
- // ▼ └───────────────────────────────────────┘
3135
- // Polling &
3136
- // instantiation
3137
- // order
3138
-
3139
- std::string DebugTag() const override {
3140
- return absl::StrFormat("SERVER_CALL[%p]: ", this);
3141
- }
3142
-
3143
- ServerCallContext* server_call_context() override { return this; }
3144
-
3145
- const void* server_stream_data() override { return server_transport_data_; }
3146
- void PublishInitialMetadata(
3147
- ClientMetadataHandle metadata,
3148
- grpc_metadata_array* publish_initial_metadata) override;
3149
- ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
3150
- CallArgs call_args, grpc_completion_queue* cq,
3151
- absl::FunctionRef<void(grpc_call* call)> publish) override;
3152
-
3153
- private:
3154
- class RecvCloseOpCancelState {
3155
- public:
3156
- // Request that receiver be filled in per
3157
- // grpc_op_recv_close_on_server. Returns true if the request can
3158
- // be fulfilled immediately. Returns false if the request will be
3159
- // fulfilled later.
3160
- bool ReceiveCloseOnServerOpStarted(int* receiver) {
3161
- uintptr_t state = state_.load(std::memory_order_acquire);
3162
- uintptr_t new_state;
3163
- do {
3164
- switch (state) {
3165
- case kUnset:
3166
- new_state = reinterpret_cast<uintptr_t>(receiver);
3167
- break;
3168
- case kFinishedWithFailure:
3169
- *receiver = 1;
3170
- return true;
3171
- case kFinishedWithSuccess:
3172
- *receiver = 0;
3173
- return true;
3174
- default:
3175
- Crash("Two threads offered ReceiveCloseOnServerOpStarted");
3176
- }
3177
- } while (!state_.compare_exchange_weak(state, new_state,
3178
- std::memory_order_acq_rel,
3179
- std::memory_order_acquire));
3180
- return false;
3181
- }
3182
-
3183
- // Mark the call as having completed.
3184
- // Returns true if this finishes a previous
3185
- // RequestReceiveCloseOnServer.
3186
- bool CompleteCallWithCancelledSetTo(bool cancelled) {
3187
- uintptr_t state = state_.load(std::memory_order_acquire);
3188
- uintptr_t new_state;
3189
- bool r;
3190
- do {
3191
- switch (state) {
3192
- case kUnset:
3193
- new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
3194
- r = false;
3195
- break;
3196
- case kFinishedWithFailure:
3197
- return false;
3198
- case kFinishedWithSuccess:
3199
- Crash("unreachable");
3200
- default:
3201
- new_state = cancelled ? kFinishedWithFailure : kFinishedWithSuccess;
3202
- r = true;
3203
- }
3204
- } while (!state_.compare_exchange_weak(state, new_state,
3205
- std::memory_order_acq_rel,
3206
- std::memory_order_acquire));
3207
- if (r) *reinterpret_cast<int*>(state) = cancelled ? 1 : 0;
3208
- return r;
3209
- }
3210
-
3211
- std::string ToString() const {
3212
- auto state = state_.load(std::memory_order_relaxed);
3213
- switch (state) {
3214
- case kUnset:
3215
- return "Unset";
3216
- case kFinishedWithFailure:
3217
- return "FinishedWithFailure";
3218
- case kFinishedWithSuccess:
3219
- return "FinishedWithSuccess";
3220
- default:
3221
- return absl::StrFormat("WaitingForReceiver(%p)",
3222
- reinterpret_cast<void*>(state));
3223
- }
3224
- }
3225
-
3226
- private:
3227
- static constexpr uintptr_t kUnset = 0;
3228
- static constexpr uintptr_t kFinishedWithFailure = 1;
3229
- static constexpr uintptr_t kFinishedWithSuccess = 2;
3230
- // Holds one of kUnset, kFinishedWithFailure, or
3231
- // kFinishedWithSuccess OR an int* that wants to receive the
3232
- // final status.
3233
- std::atomic<uintptr_t> state_{kUnset};
3234
- };
3235
-
3236
- void CommitBatch(const grpc_op* ops, size_t nops,
3237
- const Completion& completion);
3238
- void Finish(ServerMetadataHandle result);
3239
-
3240
- ServerInterface* const server_;
3241
- const void* const server_transport_data_;
3242
- PipeSender<ServerMetadataHandle>* server_initial_metadata_ = nullptr;
3243
- PipeSender<MessageHandle>* server_to_client_messages_ = nullptr;
3244
- PipeReceiver<MessageHandle>* client_to_server_messages_ = nullptr;
3245
- Latch<ServerMetadataHandle> send_trailing_metadata_;
3246
- RecvCloseOpCancelState recv_close_op_cancel_state_;
3247
- ClientMetadataHandle client_initial_metadata_;
3248
- Completion recv_close_completion_;
3249
- std::atomic<bool> cancelled_{false};
3250
- };
3251
-
3252
- ServerPromiseBasedCall::ServerPromiseBasedCall(Arena* arena,
3253
- grpc_call_create_args* args)
3254
- : PromiseBasedCall(arena, 0, *args),
3255
- server_(args->server),
3256
- server_transport_data_(args->server_transport_data) {
3257
- global_stats().IncrementServerCallsCreated();
3258
- channelz::ServerNode* channelz_node = server_->channelz_node();
3259
- if (channelz_node != nullptr) {
3260
- channelz_node->RecordCallStarted();
3261
- }
3262
- ScopedContext activity_context(this);
3263
- // TODO(yashykt): In the future, we want to also enable stats and trace
3264
- // collecting from when the call is created at the transport. The idea is that
3265
- // the transport would create the call tracer and pass it in as part of the
3266
- // metadata.
3267
- // TODO(yijiem): OpenCensus and internal Census is still using this way to
3268
- // set server call tracer. We need to refactor them to stats plugins
3269
- // (including removing the client channel filters).
3270
- if (args->server != nullptr &&
3271
- args->server->server_call_tracer_factory() != nullptr) {
3272
- auto* server_call_tracer =
3273
- args->server->server_call_tracer_factory()->CreateNewServerCallTracer(
3274
- arena, args->server->channel_args());
3275
- if (server_call_tracer != nullptr) {
3276
- // Note that we are setting both
3277
- // GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE and
3278
- // GRPC_CONTEXT_CALL_TRACER as a matter of convenience. In the future
3279
- // promise-based world, we would just a single tracer object for each
3280
- // stack (call, subchannel_call, server_call.)
3281
- ContextSet(GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE,
3282
- server_call_tracer, nullptr);
3283
- ContextSet(GRPC_CONTEXT_CALL_TRACER, server_call_tracer, nullptr);
3284
- }
3285
- }
3286
- args->channel->channel_stack()->stats_plugin_group->AddServerCallTracers(
3287
- context());
3288
- Spawn("server_promise",
3289
- channel()->channel_stack()->MakeServerCallPromise(
3290
- CallArgs{nullptr, ClientInitialMetadataOutstandingToken::Empty(),
3291
- nullptr, nullptr, nullptr, nullptr}),
3292
- [this](ServerMetadataHandle result) { Finish(std::move(result)); });
3293
- }
3294
-
3295
- void ServerPromiseBasedCall::Finish(ServerMetadataHandle result) {
3296
- if (grpc_call_trace.enabled()) {
3297
- gpr_log(GPR_INFO, "%s[call] Finish: recv_close_state:%s result:%s",
3298
- DebugTag().c_str(), recv_close_op_cancel_state_.ToString().c_str(),
3299
- result->DebugString().c_str());
3300
- }
3301
- const auto status =
3302
- result->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN);
3303
- channelz::ServerNode* channelz_node = server_->channelz_node();
3304
- if (channelz_node != nullptr) {
3305
- if (status == GRPC_STATUS_OK) {
3306
- channelz_node->RecordCallSucceeded();
3307
- } else {
3308
- channelz_node->RecordCallFailed();
3309
- }
3310
- }
3311
- bool was_cancelled = result->get(GrpcCallWasCancelled()).value_or(true);
3312
- if (recv_close_op_cancel_state_.CompleteCallWithCancelledSetTo(
3313
- was_cancelled)) {
3314
- FinishOpOnCompletion(&recv_close_completion_,
3315
- PendingOp::kReceiveCloseOnServer);
3316
- }
3317
- if (was_cancelled) set_failed_before_recv_message();
3318
- if (server_initial_metadata_ != nullptr) {
3319
- server_initial_metadata_->Close();
3320
- }
3321
- Slice message_slice;
3322
- if (Slice* message = result->get_pointer(GrpcMessageMetadata())) {
3323
- message_slice = message->Ref();
3324
- }
3325
- AcceptTransportStatsFromContext();
3326
- SetFinalizationStatus(status, std::move(message_slice));
3327
- set_completed();
3328
- ResetDeadline();
3329
- PropagateCancellationToChildren();
3330
- }
3331
-
3332
- grpc_call_error ValidateServerBatch(const grpc_op* ops, size_t nops) {
3333
- BitSet<8> got_ops;
3334
- for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3335
- const grpc_op& op = ops[op_idx];
3336
- switch (op.op) {
3337
- case GRPC_OP_SEND_INITIAL_METADATA:
3338
- if (!AreInitialMetadataFlagsValid(op.flags)) {
3339
- return GRPC_CALL_ERROR_INVALID_FLAGS;
3340
- }
3341
- if (!ValidateMetadata(op.data.send_initial_metadata.count,
3342
- op.data.send_initial_metadata.metadata)) {
3343
- return GRPC_CALL_ERROR_INVALID_METADATA;
3344
- }
3345
- break;
3346
- case GRPC_OP_SEND_MESSAGE:
3347
- if (!AreWriteFlagsValid(op.flags)) {
3348
- return GRPC_CALL_ERROR_INVALID_FLAGS;
3349
- }
3350
- break;
3351
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
3352
- if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
3353
- if (!ValidateMetadata(
3354
- op.data.send_status_from_server.trailing_metadata_count,
3355
- op.data.send_status_from_server.trailing_metadata)) {
3356
- return GRPC_CALL_ERROR_INVALID_METADATA;
3357
- }
3358
- break;
3359
- case GRPC_OP_RECV_MESSAGE:
3360
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
3361
- if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS;
3362
- break;
3363
- case GRPC_OP_RECV_INITIAL_METADATA:
3364
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
3365
- case GRPC_OP_RECV_STATUS_ON_CLIENT:
3366
- return GRPC_CALL_ERROR_NOT_ON_SERVER;
3367
- }
3368
- if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
3369
- got_ops.set(op.op);
3370
- }
3371
- return GRPC_CALL_OK;
3372
- }
3373
-
3374
- void ServerPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops,
3375
- const Completion& completion) {
3376
- Party::BulkSpawner spawner(this);
3377
- for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3378
- const grpc_op& op = ops[op_idx];
3379
- switch (op.op) {
3380
- case GRPC_OP_SEND_INITIAL_METADATA: {
3381
- auto metadata = arena()->MakePooled<ServerMetadata>();
3382
- PrepareOutgoingInitialMetadata(op, *metadata);
3383
- CToMetadata(op.data.send_initial_metadata.metadata,
3384
- op.data.send_initial_metadata.count, metadata.get());
3385
- if (grpc_call_trace.enabled()) {
3386
- gpr_log(GPR_INFO, "%s[call] Send initial metadata",
3387
- DebugTag().c_str());
3388
- }
3389
- QueueSend();
3390
- spawner.Spawn(
3391
- "call_send_initial_metadata",
3392
- [this, metadata = std::move(metadata)]() mutable {
3393
- EnactSend();
3394
- return server_initial_metadata_->Push(std::move(metadata));
3395
- },
3396
- [this,
3397
- completion = AddOpToCompletion(
3398
- completion, PendingOp::kSendInitialMetadata)](bool r) mutable {
3399
- if (!r) {
3400
- set_failed_before_recv_message();
3401
- FailCompletion(completion);
3402
- }
3403
- FinishOpOnCompletion(&completion,
3404
- PendingOp::kSendInitialMetadata);
3405
- });
3406
- } break;
3407
- case GRPC_OP_SEND_MESSAGE:
3408
- StartSendMessage(op, completion, server_to_client_messages_, spawner);
3409
- break;
3410
- case GRPC_OP_RECV_MESSAGE:
3411
- if (cancelled_.load(std::memory_order_relaxed)) {
3412
- set_failed_before_recv_message();
3413
- FailCompletion(completion);
3414
- break;
3415
- }
3416
- StartRecvMessage(
3417
- op, completion, []() { return []() { return Empty{}; }; },
3418
- client_to_server_messages_, true, spawner);
3419
- break;
3420
- case GRPC_OP_SEND_STATUS_FROM_SERVER: {
3421
- auto metadata = arena()->MakePooled<ServerMetadata>();
3422
- CToMetadata(op.data.send_status_from_server.trailing_metadata,
3423
- op.data.send_status_from_server.trailing_metadata_count,
3424
- metadata.get());
3425
- metadata->Set(GrpcStatusMetadata(),
3426
- op.data.send_status_from_server.status);
3427
- if (auto* details = op.data.send_status_from_server.status_details) {
3428
- // TODO(ctiller): this should not be a copy, but we have callers that
3429
- // allocate and pass in a slice created with
3430
- // grpc_slice_from_static_string and then delete the string after
3431
- // passing it in, which shouldn't be a supported API.
3432
- metadata->Set(GrpcMessageMetadata(),
3433
- Slice(grpc_slice_copy(*details)));
3434
- }
3435
- spawner.Spawn(
3436
- "call_send_status_from_server",
3437
- [this, metadata = std::move(metadata)]() mutable {
3438
- bool r = true;
3439
- if (send_trailing_metadata_.is_set()) {
3440
- r = false;
3441
- } else {
3442
- send_trailing_metadata_.Set(std::move(metadata));
3443
- }
3444
- return Map(WaitForSendingStarted(), [this, r](Empty) {
3445
- server_initial_metadata_->Close();
3446
- server_to_client_messages_->Close();
3447
- return r;
3448
- });
3449
- },
3450
- [this, completion = AddOpToCompletion(
3451
- completion, PendingOp::kSendStatusFromServer)](
3452
- bool ok) mutable {
3453
- if (!ok) {
3454
- set_failed_before_recv_message();
3455
- FailCompletion(completion);
3456
- }
3457
- FinishOpOnCompletion(&completion,
3458
- PendingOp::kSendStatusFromServer);
3459
- });
3460
- } break;
3461
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
3462
- if (grpc_call_trace.enabled()) {
3463
- gpr_log(GPR_INFO, "%s[call] StartBatch: RecvClose %s",
3464
- DebugTag().c_str(),
3465
- recv_close_op_cancel_state_.ToString().c_str());
3466
- }
3467
- ForceCompletionSuccess(completion);
3468
- recv_close_completion_ =
3469
- AddOpToCompletion(completion, PendingOp::kReceiveCloseOnServer);
3470
- if (recv_close_op_cancel_state_.ReceiveCloseOnServerOpStarted(
3471
- op.data.recv_close_on_server.cancelled)) {
3472
- FinishOpOnCompletion(&recv_close_completion_,
3473
- PendingOp::kReceiveCloseOnServer);
3474
- }
3475
- break;
3476
- case GRPC_OP_RECV_STATUS_ON_CLIENT:
3477
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
3478
- case GRPC_OP_RECV_INITIAL_METADATA:
3479
- abort(); // unreachable
3480
- }
3481
- }
3482
- }
3483
-
3484
- grpc_call_error ServerPromiseBasedCall::StartBatch(const grpc_op* ops,
3485
- size_t nops,
3486
- void* notify_tag,
3487
- bool is_notify_tag_closure) {
3488
- if (nops == 0) {
3489
- EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
3490
- return GRPC_CALL_OK;
3491
- }
3492
- const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
3493
- if (validation_result != GRPC_CALL_OK) {
3494
- return validation_result;
3495
- }
3496
- Completion completion =
3497
- StartCompletion(notify_tag, is_notify_tag_closure, ops);
3498
- CommitBatch(ops, nops, completion);
3499
- FinishOpOnCompletion(&completion, PendingOp::kStartingBatch);
3500
- return GRPC_CALL_OK;
3501
- }
3502
-
3503
- void ServerPromiseBasedCall::CancelWithError(absl::Status error) {
3504
- cancelled_.store(true, std::memory_order_relaxed);
3505
- Spawn(
3506
- "cancel_with_error",
3507
- [this, error = std::move(error)]() {
3508
- if (!send_trailing_metadata_.is_set()) {
3509
- auto md = ServerMetadataFromStatus(error);
3510
- md->Set(GrpcCallWasCancelled(), true);
3511
- send_trailing_metadata_.Set(std::move(md));
3512
- }
3513
- if (server_to_client_messages_ != nullptr) {
3514
- server_to_client_messages_->Close();
3515
- }
3516
- if (server_initial_metadata_ != nullptr) {
3517
- server_initial_metadata_->Close();
3518
- }
3519
- return Empty{};
3520
- },
3521
- [](Empty) {});
3522
- }
3523
- #endif
3524
-
3525
- #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
3526
- void ServerPromiseBasedCall::PublishInitialMetadata(
3527
- ClientMetadataHandle metadata,
3528
- grpc_metadata_array* publish_initial_metadata) {
3529
- if (grpc_call_trace.enabled()) {
3530
- gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
3531
- metadata->DebugString().c_str());
3532
- }
3533
- PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
3534
- client_initial_metadata_ = std::move(metadata);
3535
- }
3536
-
3537
- ArenaPromise<ServerMetadataHandle>
3538
- ServerPromiseBasedCall::MakeTopOfServerCallPromise(
3539
- CallArgs call_args, grpc_completion_queue* cq,
3540
- absl::FunctionRef<void(grpc_call* call)> publish) {
3541
- SetCompletionQueue(cq);
3542
- call_args.polling_entity->Set(
3543
- grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)));
3544
- server_to_client_messages_ = call_args.server_to_client_messages;
3545
- client_to_server_messages_ = call_args.client_to_server_messages;
3546
- server_initial_metadata_ = call_args.server_initial_metadata;
3547
- absl::optional<Timestamp> deadline =
3548
- client_initial_metadata_->get(GrpcTimeoutMetadata());
3549
- if (deadline.has_value()) {
3550
- set_send_deadline(*deadline);
3551
- UpdateDeadline(*deadline);
3552
- }
3553
- ProcessIncomingInitialMetadata(*client_initial_metadata_);
3554
- ExternalRef();
3555
- publish(c_ptr());
3556
- return Seq(server_to_client_messages_->AwaitClosed(),
3557
- send_trailing_metadata_.Wait());
3558
- }
3559
-
3560
- ///////////////////////////////////////////////////////////////////////////////
3561
- // CallSpine based Server Call
3562
-
3563
- class ServerCallSpine final : public PipeBasedCallSpine,
3564
- public ServerCallContext,
3565
- public BasicPromiseBasedCall {
3566
- public:
3567
- ServerCallSpine(ClientMetadataHandle client_initial_metadata,
3568
- ServerInterface* server, Channel* channel, Arena* arena);
3569
-
3570
- // CallSpineInterface
3571
- Pipe<ClientMetadataHandle>& client_initial_metadata() override {
3572
- return client_initial_metadata_;
3573
- }
3574
- Pipe<ServerMetadataHandle>& server_initial_metadata() override {
3575
- return server_initial_metadata_;
3576
- }
3577
- Pipe<MessageHandle>& client_to_server_messages() override {
3578
- return client_to_server_messages_;
3579
- }
3580
- Pipe<MessageHandle>& server_to_client_messages() override {
3581
- return server_to_client_messages_;
3582
- }
3583
- Latch<ServerMetadataHandle>& cancel_latch() override { return cancel_latch_; }
3584
- Latch<bool>& was_cancelled_latch() override { return was_cancelled_latch_; }
3585
- Party& party() override { return *this; }
3586
- Arena* arena() override { return BasicPromiseBasedCall::arena(); }
3587
- void IncrementRefCount() override { InternalRef("CallSpine"); }
3588
- void Unref() override { InternalUnref("CallSpine"); }
3589
-
3590
- // PromiseBasedCall
3591
- void OrphanCall() override {
3592
- ResetDeadline();
3593
- CancelWithError(absl::CancelledError());
3594
- }
3595
- void CancelWithError(grpc_error_handle error) override {
3596
- SpawnInfallible("CancelWithError", [this, error = std::move(error)] {
3597
- auto status = ServerMetadataFromStatus(error);
3598
- status->Set(GrpcCallWasCancelled(), true);
3599
- PushServerTrailingMetadata(std::move(status));
3600
- return Empty{};
3601
- });
3602
- }
3603
- bool is_trailers_only() const override {
3604
- Crash("is_trailers_only not implemented for server calls");
3605
- }
3606
- absl::string_view GetServerAuthority() const override {
3607
- Crash("unimplemented");
3608
- }
3609
- grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag,
3610
- bool is_notify_tag_closure) override;
3611
-
3612
- bool Completed() final { Crash("unimplemented"); }
3613
- bool failed_before_recv_message() const final { Crash("unimplemented"); }
3614
-
3615
- ServerCallContext* server_call_context() override { return this; }
3616
- const void* server_stream_data() override { Crash("unimplemented"); }
3617
- void PublishInitialMetadata(
3618
- ClientMetadataHandle metadata,
3619
- grpc_metadata_array* publish_initial_metadata) override;
3620
- ArenaPromise<ServerMetadataHandle> MakeTopOfServerCallPromise(
3621
- CallArgs, grpc_completion_queue*,
3622
- absl::FunctionRef<void(grpc_call* call)>) override {
3623
- Crash("unimplemented");
3624
- }
3625
-
3626
- void V2HackToStartCallWithoutACallFilterStack() override {}
3627
-
3628
- ClientMetadata& UnprocessedClientInitialMetadata() override {
3629
- Crash("not for v2");
3630
- }
3631
-
3632
- bool RunParty() override {
3633
- ScopedContext ctx(this);
3634
- return Party::RunParty();
3635
- }
3636
-
3637
- private:
3638
- void CommitBatch(const grpc_op* ops, size_t nops, void* notify_tag,
3639
- bool is_notify_tag_closure);
3640
- StatusFlag FinishRecvMessage(NextResult<MessageHandle> result);
3641
-
3642
- std::string DebugTag() const override {
3643
- return absl::StrFormat("SERVER_CALL_SPINE[%p]: ", this);
3644
- }
3645
-
3646
- // Initial metadata from client to server
3647
- Pipe<ClientMetadataHandle> client_initial_metadata_;
3648
- // Initial metadata from server to client
3649
- Pipe<ServerMetadataHandle> server_initial_metadata_;
3650
- // Messages travelling from the application to the transport.
3651
- Pipe<MessageHandle> client_to_server_messages_;
3652
- // Messages travelling from the transport to the application.
3653
- Pipe<MessageHandle> server_to_client_messages_;
3654
- // Latch that can be set to terminate the call
3655
- Latch<ServerMetadataHandle> cancel_latch_;
3656
- Latch<bool> was_cancelled_latch_;
3657
- grpc_byte_buffer** recv_message_ = nullptr;
3658
- ClientMetadataHandle client_initial_metadata_stored_;
3659
- };
3660
-
3661
- ServerCallSpine::ServerCallSpine(ClientMetadataHandle client_initial_metadata,
3662
- ServerInterface* server, Channel* channel,
3663
- Arena* arena)
3664
- : BasicPromiseBasedCall(arena, 0, 1,
3665
- [channel, server]() -> grpc_call_create_args {
3666
- grpc_call_create_args args;
3667
- args.channel = channel->Ref();
3668
- args.server = server;
3669
- args.parent = nullptr;
3670
- args.propagation_mask = 0;
3671
- args.cq = nullptr;
3672
- args.pollset_set_alternative = nullptr;
3673
- args.server_transport_data =
3674
- &args; // Arbitrary non-null pointer
3675
- args.send_deadline = Timestamp::InfFuture();
3676
- return args;
3677
- }()),
3678
- client_initial_metadata_(arena),
3679
- server_initial_metadata_(arena),
3680
- client_to_server_messages_(arena),
3681
- server_to_client_messages_(arena) {
3682
- global_stats().IncrementServerCallsCreated();
3683
- ScopedContext ctx(this);
3684
- channel->channel_stack()->InitServerCallSpine(this);
3685
- SpawnGuarded("push_client_initial_metadata",
3686
- [this, md = std::move(client_initial_metadata)]() mutable {
3687
- return Map(client_initial_metadata_.sender.Push(std::move(md)),
3688
- [](bool r) { return StatusFlag(r); });
3689
- });
3690
- }
3691
-
3692
- void ServerCallSpine::PublishInitialMetadata(
3693
- ClientMetadataHandle metadata,
3694
- grpc_metadata_array* publish_initial_metadata) {
3695
- if (grpc_call_trace.enabled()) {
3696
- gpr_log(GPR_INFO, "%s[call] PublishInitialMetadata: %s", DebugTag().c_str(),
3697
- metadata->DebugString().c_str());
3698
- }
3699
- PublishMetadataArray(metadata.get(), publish_initial_metadata, false);
3700
- client_initial_metadata_stored_ = std::move(metadata);
3701
- }
3702
-
3703
- grpc_call_error ServerCallSpine::StartBatch(const grpc_op* ops, size_t nops,
3704
- void* notify_tag,
3705
- bool is_notify_tag_closure) {
3706
- if (nops == 0) {
3707
- EndOpImmediately(cq(), notify_tag, is_notify_tag_closure);
3708
- return GRPC_CALL_OK;
3709
- }
3710
- const grpc_call_error validation_result = ValidateServerBatch(ops, nops);
3711
- if (validation_result != GRPC_CALL_OK) {
3712
- return validation_result;
3713
- }
3714
- CommitBatch(ops, nops, notify_tag, is_notify_tag_closure);
3715
- return GRPC_CALL_OK;
3716
- }
3717
-
3718
- namespace {
3719
- template <typename SetupFn>
3720
- class MaybeOpImpl {
3721
- public:
3722
- using SetupResult = decltype(std::declval<SetupFn>()(grpc_op()));
3723
- using PromiseFactory = promise_detail::OncePromiseFactory<void, SetupResult>;
3724
- using Promise = typename PromiseFactory::Promise;
3725
- struct Dismissed {};
3726
- using State = absl::variant<Dismissed, PromiseFactory, Promise>;
3727
-
3728
- // op_ is garbage but shouldn't be uninitialized
3729
- MaybeOpImpl() : state_(Dismissed{}), op_(GRPC_OP_RECV_STATUS_ON_CLIENT) {}
3730
- MaybeOpImpl(SetupResult result, grpc_op_type op)
3731
- : state_(PromiseFactory(std::move(result))), op_(op) {}
3732
-
3733
- MaybeOpImpl(const MaybeOpImpl&) = delete;
3734
- MaybeOpImpl& operator=(const MaybeOpImpl&) = delete;
3735
- MaybeOpImpl(MaybeOpImpl&& other) noexcept
3736
- : state_(MoveState(other.state_)), op_(other.op_) {}
3737
- MaybeOpImpl& operator=(MaybeOpImpl&& other) noexcept {
3738
- op_ = other.op_;
3739
- if (absl::holds_alternative<Dismissed>(state_)) {
3740
- state_.template emplace<Dismissed>();
3741
- return *this;
3742
- }
3743
- // Can't move after first poll => Promise is not an option
3744
- state_.template emplace<PromiseFactory>(
3745
- std::move(absl::get<PromiseFactory>(other.state_)));
3746
- return *this;
3747
- }
3748
-
3749
- Poll<StatusFlag> operator()() {
3750
- if (absl::holds_alternative<Dismissed>(state_)) return Success{};
3751
- if (absl::holds_alternative<PromiseFactory>(state_)) {
3752
- auto& factory = absl::get<PromiseFactory>(state_);
3753
- auto promise = factory.Make();
3754
- state_.template emplace<Promise>(std::move(promise));
3755
- }
3756
- if (grpc_call_trace.enabled()) {
3757
- gpr_log(GPR_INFO, "%sBeginPoll %s",
3758
- Activity::current()->DebugTag().c_str(), OpName(op_).c_str());
3759
- }
3760
- auto& promise = absl::get<Promise>(state_);
3761
- auto r = poll_cast<StatusFlag>(promise());
3762
- if (grpc_call_trace.enabled()) {
3763
- gpr_log(GPR_INFO, "%sEndPoll %s --> %s",
3764
- Activity::current()->DebugTag().c_str(), OpName(op_).c_str(),
3765
- r.pending() ? "PENDING" : (r.value().ok() ? "OK" : "FAILURE"));
3766
- }
3767
- return r;
3768
- }
3769
-
3770
- private:
3771
- GPR_NO_UNIQUE_ADDRESS State state_;
3772
- GPR_NO_UNIQUE_ADDRESS grpc_op_type op_;
3773
-
3774
- static std::string OpName(grpc_op_type op) {
3775
- switch (op) {
3776
- case GRPC_OP_SEND_INITIAL_METADATA:
3777
- return "SendInitialMetadata";
3778
- case GRPC_OP_SEND_MESSAGE:
3779
- return "SendMessage";
3780
- case GRPC_OP_SEND_STATUS_FROM_SERVER:
3781
- return "SendStatusFromServer";
3782
- case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
3783
- return "SendCloseFromClient";
3784
- case GRPC_OP_RECV_MESSAGE:
3785
- return "RecvMessage";
3786
- case GRPC_OP_RECV_CLOSE_ON_SERVER:
3787
- return "RecvCloseOnServer";
3788
- case GRPC_OP_RECV_INITIAL_METADATA:
3789
- return "RecvInitialMetadata";
3790
- case GRPC_OP_RECV_STATUS_ON_CLIENT:
3791
- return "RecvStatusOnClient";
3792
- }
3793
- return absl::StrCat("UnknownOp(", op, ")");
3794
- }
3795
-
3796
- static State MoveState(State& state) {
3797
- if (absl::holds_alternative<Dismissed>(state)) return Dismissed{};
3798
- // Can't move after first poll => Promise is not an option
3799
- return std::move(absl::get<PromiseFactory>(state));
3800
- }
3801
- };
3802
-
3803
- // MaybeOp captures a fairly complicated dance we need to do for the batch API.
3804
- // We first check if an op is included or not, and if it is, we run the setup
3805
- // function in the context of the API call (NOT in the call party).
3806
- // This setup function returns a promise factory which we'll then run *in* the
3807
- // party to do initial setup, and have it return the promise that we'll
3808
- // ultimately poll on til completion.
3809
- // Once we express our surface API in terms of core internal types this whole
3810
- // dance will go away.
3811
- template <typename SetupFn>
3812
- auto MaybeOp(const grpc_op* ops, uint8_t idx, SetupFn setup) {
3813
- if (idx == 255) {
3814
- return MaybeOpImpl<SetupFn>();
3815
- } else {
3816
- return MaybeOpImpl<SetupFn>(setup(ops[idx]), ops[idx].op);
3817
- }
3818
- }
3819
-
3820
- template <typename F>
3821
- class PollBatchLogger {
3822
- public:
3823
- PollBatchLogger(void* tag, F f) : tag_(tag), f_(std::move(f)) {}
3824
-
3825
- auto operator()() {
3826
- if (grpc_call_trace.enabled()) {
3827
- gpr_log(GPR_INFO, "Poll batch %p", tag_);
3828
- }
3829
- auto r = f_();
3830
- if (grpc_call_trace.enabled()) {
3831
- gpr_log(GPR_INFO, "Poll batch %p --> %s", tag_, ResultString(r).c_str());
3832
- }
3833
- return r;
3834
- }
3835
-
3836
- private:
3837
- template <typename T>
3838
- static std::string ResultString(Poll<T> r) {
3839
- if (r.pending()) return "PENDING";
3840
- return ResultString(r.value());
3841
- }
3842
- static std::string ResultString(Empty) { return "DONE"; }
3843
-
3844
- void* tag_;
3845
- F f_;
3846
- };
3847
-
3848
- template <typename F>
3849
- PollBatchLogger<F> LogPollBatch(void* tag, F f) {
3850
- return PollBatchLogger<F>(tag, std::move(f));
3851
- }
3852
- } // namespace
3853
-
3854
- StatusFlag ServerCallSpine::FinishRecvMessage(
3855
- NextResult<MessageHandle> result) {
3856
- if (result.has_value()) {
3857
- MessageHandle& message = *result;
3858
- NoteLastMessageFlags(message->flags());
3859
- if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) &&
3860
- (incoming_compression_algorithm() != GRPC_COMPRESS_NONE)) {
3861
- *recv_message_ = grpc_raw_compressed_byte_buffer_create(
3862
- nullptr, 0, incoming_compression_algorithm());
3863
- } else {
3864
- *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0);
3865
- }
3866
- grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(),
3867
- &(*recv_message_)->data.raw.slice_buffer);
3868
- if (grpc_call_trace.enabled()) {
3869
- gpr_log(GPR_INFO,
3870
- "%s[call] RecvMessage: outstanding_recv "
3871
- "finishes: received %" PRIdPTR " byte message",
3872
- DebugTag().c_str(),
3873
- (*recv_message_)->data.raw.slice_buffer.length);
3874
- }
3875
- recv_message_ = nullptr;
3876
- return Success{};
3877
- }
3878
- if (result.cancelled()) {
3879
- if (grpc_call_trace.enabled()) {
3880
- gpr_log(GPR_INFO,
3881
- "%s[call] RecvMessage: outstanding_recv "
3882
- "finishes: received end-of-stream with error",
3883
- DebugTag().c_str());
3884
- }
3885
- *recv_message_ = nullptr;
3886
- recv_message_ = nullptr;
3887
- return Failure{};
3888
- }
3889
- if (grpc_call_trace.enabled()) {
3890
- gpr_log(GPR_INFO,
3891
- "%s[call] RecvMessage: outstanding_recv "
3892
- "finishes: received end-of-stream",
3893
- DebugTag().c_str());
3894
- }
3895
- *recv_message_ = nullptr;
3896
- recv_message_ = nullptr;
3897
- return Success{};
3898
- }
3899
-
3900
- void ServerCallSpine::CommitBatch(const grpc_op* ops, size_t nops,
3901
- void* notify_tag,
3902
- bool is_notify_tag_closure) {
3903
- std::array<uint8_t, 8> got_ops{255, 255, 255, 255, 255, 255, 255, 255};
3904
- for (size_t op_idx = 0; op_idx < nops; op_idx++) {
3905
- const grpc_op& op = ops[op_idx];
3906
- got_ops[op.op] = op_idx;
3907
- }
3908
- if (!is_notify_tag_closure) grpc_cq_begin_op(cq(), notify_tag);
3909
- auto send_initial_metadata = MaybeOp(
3910
- ops, got_ops[GRPC_OP_SEND_INITIAL_METADATA], [this](const grpc_op& op) {
3911
- auto metadata = arena()->MakePooled<ServerMetadata>();
3912
- PrepareOutgoingInitialMetadata(op, *metadata);
3913
- CToMetadata(op.data.send_initial_metadata.metadata,
3914
- op.data.send_initial_metadata.count, metadata.get());
3915
- if (grpc_call_trace.enabled()) {
3916
- gpr_log(GPR_INFO, "%s[call] Send initial metadata",
3917
- DebugTag().c_str());
3918
- }
3919
- return [this, metadata = std::move(metadata)]() mutable {
3920
- return Map(server_initial_metadata_.sender.Push(std::move(metadata)),
3921
- [this](bool r) {
3922
- server_initial_metadata_.sender.Close();
3923
- return StatusFlag(r);
3924
- });
3925
- };
3926
- });
3927
- auto send_message =
3928
- MaybeOp(ops, got_ops[GRPC_OP_SEND_MESSAGE], [this](const grpc_op& op) {
3929
- SliceBuffer send;
3930
- grpc_slice_buffer_swap(
3931
- &op.data.send_message.send_message->data.raw.slice_buffer,
3932
- send.c_slice_buffer());
3933
- auto msg = arena()->MakePooled<Message>(std::move(send), op.flags);
3934
- return [this, msg = std::move(msg)]() mutable {
3935
- return Map(server_to_client_messages_.sender.Push(std::move(msg)),
3936
- [](bool r) { return StatusFlag(r); });
3937
- };
3938
- });
3939
- auto send_trailing_metadata = MaybeOp(
3940
- ops, got_ops[GRPC_OP_SEND_STATUS_FROM_SERVER], [this](const grpc_op& op) {
3941
- auto metadata = arena()->MakePooled<ServerMetadata>();
3942
- CToMetadata(op.data.send_status_from_server.trailing_metadata,
3943
- op.data.send_status_from_server.trailing_metadata_count,
3944
- metadata.get());
3945
- metadata->Set(GrpcStatusMetadata(),
3946
- op.data.send_status_from_server.status);
3947
- if (auto* details = op.data.send_status_from_server.status_details) {
3948
- // TODO(ctiller): this should not be a copy, but we have
3949
- // callers that allocate and pass in a slice created with
3950
- // grpc_slice_from_static_string and then delete the string
3951
- // after passing it in, which shouldn't be a supported API.
3952
- metadata->Set(GrpcMessageMetadata(),
3953
- Slice(grpc_slice_copy(*details)));
3954
- }
3955
- CHECK(metadata != nullptr);
3956
- return [this, metadata = std::move(metadata)]() mutable {
3957
- CHECK(metadata != nullptr);
3958
- return [this,
3959
- metadata = std::move(metadata)]() mutable -> Poll<Success> {
3960
- CHECK(metadata != nullptr);
3961
- PushServerTrailingMetadata(std::move(metadata));
3962
- return Success{};
3963
- };
3964
- };
3965
- });
3966
- auto recv_message =
3967
- MaybeOp(ops, got_ops[GRPC_OP_RECV_MESSAGE], [this](const grpc_op& op) {
3968
- CHECK_EQ(recv_message_, nullptr);
3969
- recv_message_ = op.data.recv_message.recv_message;
3970
- return [this]() mutable {
3971
- return Map(client_to_server_messages_.receiver.Next(),
3972
- [this](NextResult<MessageHandle> msg) {
3973
- return FinishRecvMessage(std::move(msg));
3974
- });
3975
- };
3976
- });
3977
- auto primary_ops = AllOk<StatusFlag>(
3978
- TrySeq(AllOk<StatusFlag>(std::move(send_initial_metadata),
3979
- std::move(send_message)),
3980
- std::move(send_trailing_metadata)),
3981
- std::move(recv_message));
3982
- if (got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER] != 255) {
3983
- auto recv_trailing_metadata = MaybeOp(
3984
- ops, got_ops[GRPC_OP_RECV_CLOSE_ON_SERVER], [this](const grpc_op& op) {
3985
- return [this, cancelled = op.data.recv_close_on_server.cancelled]() {
3986
- return Map(WasCancelled(),
3987
- [cancelled, this](bool result) -> Success {
3988
- ResetDeadline();
3989
- *cancelled = result ? 1 : 0;
3990
- return Success{};
3991
- });
3992
- };
3993
- });
3994
- SpawnInfallible(
3995
- "final-batch",
3996
- [primary_ops = std::move(primary_ops),
3997
- recv_trailing_metadata = std::move(recv_trailing_metadata),
3998
- is_notify_tag_closure, notify_tag, this]() mutable {
3999
- return LogPollBatch(
4000
- notify_tag,
4001
- Seq(std::move(primary_ops), std::move(recv_trailing_metadata),
4002
- [is_notify_tag_closure, notify_tag, this](StatusFlag) {
4003
- return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
4004
- absl::OkStatus(), cq());
4005
- }));
4006
- });
4007
- } else {
4008
- SpawnInfallible("batch", [primary_ops = std::move(primary_ops),
4009
- is_notify_tag_closure, notify_tag,
4010
- this]() mutable {
4011
- return LogPollBatch(
4012
- notify_tag,
4013
- Seq(std::move(primary_ops),
4014
- [is_notify_tag_closure, notify_tag, this](StatusFlag r) {
4015
- return WaitForCqEndOp(is_notify_tag_closure, notify_tag,
4016
- StatusCast<grpc_error_handle>(r), cq());
4017
- }));
4018
- });
4019
- }
4020
- }
4021
-
4022
- RefCountedPtr<CallSpineInterface> MakeServerCall(
4023
- ClientMetadataHandle client_initial_metadata, ServerInterface* server,
4024
- Channel* channel, Arena* arena) {
4025
- return RefCountedPtr<ServerCallSpine>(arena->New<ServerCallSpine>(
4026
- std::move(client_initial_metadata), server, channel, arena));
4027
- }
4028
- #else
4029
- RefCountedPtr<CallSpineInterface> MakeServerCall(ClientMetadataHandle,
4030
- ServerInterface*, Channel*,
4031
- Arena*) {
4032
- Crash("not implemented");
4033
- }
4034
- #endif
4035
-
4036
- } // namespace grpc_core
4037
-
4038
- ///////////////////////////////////////////////////////////////////////////////
4039
- // C-based API
4040
-
4041
- void* grpc_call_arena_alloc(grpc_call* call, size_t size) {
4042
- grpc_core::ExecCtx exec_ctx;
4043
- return grpc_core::Call::FromC(call)->arena()->Alloc(size);
4044
- }
4045
-
4046
- size_t grpc_call_get_initial_size_estimate() {
4047
- return grpc_core::FilterStackCall::InitialSizeEstimate();
4048
- }
4049
-
4050
- grpc_error_handle grpc_call_create(grpc_call_create_args* args,
4051
- grpc_call** out_call) {
4052
- #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
4053
- if (grpc_core::IsPromiseBasedClientCallEnabled() &&
4054
- args->server_transport_data == nullptr && args->channel->is_promising()) {
4055
- return grpc_core::MakePromiseBasedCall<grpc_core::ClientPromiseBasedCall>(
4056
- args, out_call);
4057
- }
4058
- #endif
4059
- #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
4060
- if (grpc_core::IsPromiseBasedServerCallEnabled() &&
4061
- args->server_transport_data != nullptr && args->channel->is_promising()) {
4062
- return grpc_core::MakePromiseBasedCall<grpc_core::ServerPromiseBasedCall>(
4063
- args, out_call);
4064
- }
4065
- #endif
4066
- return grpc_core::FilterStackCall::Create(args, out_call);
4067
- }
4068
-
4069
- void grpc_call_set_completion_queue(grpc_call* call,
4070
- grpc_completion_queue* cq) {
4071
- grpc_core::Call::FromC(call)->SetCompletionQueue(cq);
4072
- }
4073
-
4074
- void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); }
4075
-
4076
- void grpc_call_unref(grpc_call* c) {
4077
- grpc_core::ExecCtx exec_ctx;
4078
- grpc_core::Call::FromC(c)->ExternalUnref();
401
+ void grpc_call_unref(grpc_call* c) {
402
+ grpc_core::ExecCtx exec_ctx;
403
+ grpc_core::Call::FromC(c)->ExternalUnref();
4079
404
  }
4080
405
 
4081
406
  char* grpc_call_get_peer(grpc_call* call) {
4082
407
  return grpc_core::Call::FromC(call)->GetPeer();
4083
408
  }
4084
409
 
4085
- grpc_call* grpc_call_from_top_element(grpc_call_element* surface_element) {
4086
- return grpc_core::FilterStackCall::FromTopElem(surface_element)->c_ptr();
4087
- }
4088
-
4089
410
  grpc_call_error grpc_call_cancel(grpc_call* call, void* reserved) {
4090
411
  GRPC_API_TRACE("grpc_call_cancel(call=%p, reserved=%p)", 2, (call, reserved));
4091
412
  CHECK_EQ(reserved, nullptr);
@@ -4122,7 +443,7 @@ void grpc_call_cancel_internal(grpc_call* call) {
4122
443
 
4123
444
  grpc_compression_algorithm grpc_call_test_only_get_compression_algorithm(
4124
445
  grpc_call* call) {
4125
- return grpc_core::Call::FromC(call)->test_only_compression_algorithm();
446
+ return grpc_core::Call::FromC(call)->incoming_compression_algorithm();
4126
447
  }
4127
448
 
4128
449
  uint32_t grpc_call_test_only_get_message_flags(grpc_call* call) {
@@ -4166,13 +487,17 @@ grpc_call_error grpc_call_start_batch_and_execute(grpc_call* call,
4166
487
  return grpc_core::Call::FromC(call)->StartBatch(ops, nops, closure, true);
4167
488
  }
4168
489
 
4169
- void grpc_call_context_set(grpc_call* call, grpc_context_index elem,
4170
- void* value, void (*destroy)(void* value)) {
4171
- return grpc_core::Call::FromC(call)->ContextSet(elem, value, destroy);
490
+ void grpc_call_tracer_set(grpc_call* call,
491
+ grpc_core::ClientCallTracer* tracer) {
492
+ grpc_core::Arena* arena = grpc_call_get_arena(call);
493
+ return arena->SetContext<grpc_core::CallTracerAnnotationInterface>(tracer);
4172
494
  }
4173
495
 
4174
- void* grpc_call_context_get(grpc_call* call, grpc_context_index elem) {
4175
- return grpc_core::Call::FromC(call)->ContextGet(elem);
496
+ void* grpc_call_tracer_get(grpc_call* call) {
497
+ grpc_core::Arena* arena = grpc_call_get_arena(call);
498
+ auto* call_tracer =
499
+ arena->GetContext<grpc_core::CallTracerAnnotationInterface>();
500
+ return call_tracer;
4176
501
  }
4177
502
 
4178
503
  uint8_t grpc_call_is_client(grpc_call* call) {