grpc 1.62.0 → 1.69.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (2831) hide show
  1. checksums.yaml +4 -4
  2. data/Makefile +552 -2066
  3. data/include/grpc/byte_buffer.h +1 -2
  4. data/include/grpc/census.h +1 -2
  5. data/include/grpc/compression.h +2 -4
  6. data/include/grpc/credentials.h +1221 -0
  7. data/include/grpc/event_engine/README.md +1 -1
  8. data/include/grpc/event_engine/event_engine.h +41 -15
  9. data/include/grpc/event_engine/extensible.h +3 -0
  10. data/include/grpc/event_engine/internal/memory_allocator_impl.h +2 -3
  11. data/include/grpc/event_engine/internal/slice_cast.h +1 -1
  12. data/include/grpc/event_engine/memory_allocator.h +2 -4
  13. data/include/grpc/event_engine/memory_request.h +18 -1
  14. data/include/grpc/event_engine/slice.h +2 -5
  15. data/include/grpc/event_engine/slice_buffer.h +5 -8
  16. data/include/grpc/grpc.h +9 -10
  17. data/include/grpc/grpc_audit_logging.h +1 -2
  18. data/include/grpc/grpc_crl_provider.h +4 -3
  19. data/include/grpc/grpc_cronet.h +1 -2
  20. data/include/grpc/grpc_posix.h +2 -4
  21. data/include/grpc/grpc_security.h +2 -1175
  22. data/include/grpc/impl/call.h +1 -2
  23. data/include/grpc/impl/channel_arg_names.h +14 -7
  24. data/include/grpc/impl/grpc_types.h +2 -4
  25. data/include/grpc/impl/slice_type.h +0 -1
  26. data/include/grpc/module.modulemap +3 -0
  27. data/include/grpc/passive_listener.h +62 -0
  28. data/include/grpc/slice.h +1 -2
  29. data/include/grpc/slice_buffer.h +1 -2
  30. data/include/grpc/support/alloc.h +0 -1
  31. data/include/grpc/support/json.h +0 -1
  32. data/include/grpc/support/log.h +36 -63
  33. data/include/grpc/support/metrics.h +66 -0
  34. data/include/grpc/support/port_platform.h +31 -1
  35. data/include/grpc/support/string_util.h +0 -1
  36. data/include/grpc/support/sync.h +0 -1
  37. data/include/grpc/support/sync_abseil.h +0 -1
  38. data/include/grpc/support/sync_custom.h +0 -1
  39. data/include/grpc/support/sync_generic.h +3 -6
  40. data/include/grpc/support/sync_posix.h +1 -3
  41. data/include/grpc/support/time.h +0 -1
  42. data/src/core/channelz/channel_trace.cc +180 -0
  43. data/src/core/channelz/channel_trace.h +138 -0
  44. data/src/core/channelz/channelz.cc +670 -0
  45. data/src/core/channelz/channelz.h +421 -0
  46. data/src/core/channelz/channelz_registry.cc +281 -0
  47. data/src/core/channelz/channelz_registry.h +101 -0
  48. data/src/core/client_channel/backup_poller.cc +25 -16
  49. data/src/core/client_channel/client_channel.cc +1426 -0
  50. data/src/core/client_channel/client_channel.h +245 -0
  51. data/src/core/client_channel/client_channel_factory.cc +2 -2
  52. data/src/core/client_channel/client_channel_factory.h +1 -2
  53. data/src/core/client_channel/client_channel_filter.cc +437 -944
  54. data/src/core/client_channel/client_channel_filter.h +30 -93
  55. data/src/core/client_channel/client_channel_internal.h +27 -12
  56. data/src/core/client_channel/client_channel_plugin.cc +3 -18
  57. data/src/core/client_channel/client_channel_service_config.cc +2 -3
  58. data/src/core/client_channel/client_channel_service_config.h +12 -13
  59. data/src/core/client_channel/config_selector.h +26 -26
  60. data/src/core/client_channel/connector.h +4 -4
  61. data/src/core/client_channel/direct_channel.cc +83 -0
  62. data/src/core/client_channel/direct_channel.h +101 -0
  63. data/src/core/client_channel/dynamic_filters.cc +19 -16
  64. data/src/core/client_channel/dynamic_filters.h +10 -11
  65. data/src/core/client_channel/global_subchannel_pool.cc +2 -2
  66. data/src/core/client_channel/global_subchannel_pool.h +2 -3
  67. data/src/core/client_channel/lb_metadata.cc +120 -0
  68. data/src/core/client_channel/lb_metadata.h +55 -0
  69. data/src/core/client_channel/load_balanced_call_destination.cc +273 -0
  70. data/src/core/client_channel/load_balanced_call_destination.h +48 -0
  71. data/src/core/client_channel/local_subchannel_pool.cc +6 -7
  72. data/src/core/client_channel/local_subchannel_pool.h +1 -1
  73. data/src/core/client_channel/retry_filter.cc +8 -17
  74. data/src/core/client_channel/retry_filter.h +10 -18
  75. data/src/core/client_channel/retry_filter_legacy_call_data.cc +215 -324
  76. data/src/core/client_channel/retry_filter_legacy_call_data.h +20 -23
  77. data/src/core/client_channel/retry_service_config.cc +10 -13
  78. data/src/core/client_channel/retry_service_config.h +9 -11
  79. data/src/core/client_channel/retry_throttle.cc +3 -4
  80. data/src/core/client_channel/retry_throttle.h +7 -9
  81. data/src/core/client_channel/subchannel.cc +295 -176
  82. data/src/core/client_channel/subchannel.h +53 -46
  83. data/src/core/client_channel/subchannel_interface_internal.h +1 -1
  84. data/src/core/client_channel/subchannel_pool_interface.cc +1 -5
  85. data/src/core/client_channel/subchannel_pool_interface.h +5 -8
  86. data/src/core/client_channel/subchannel_stream_client.cc +47 -65
  87. data/src/core/client_channel/subchannel_stream_client.h +12 -16
  88. data/src/core/config/config_vars.cc +151 -0
  89. data/src/core/config/config_vars.h +128 -0
  90. data/src/core/config/config_vars_non_generated.cc +49 -0
  91. data/src/core/config/core_configuration.cc +111 -0
  92. data/src/core/config/core_configuration.h +242 -0
  93. data/src/core/config/load_config.cc +77 -0
  94. data/src/core/config/load_config.h +54 -0
  95. data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +30 -80
  96. data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +5 -15
  97. data/src/core/ext/filters/backend_metrics/backend_metric_provider.h +7 -0
  98. data/src/core/ext/filters/census/grpc_context.cc +7 -10
  99. data/src/core/ext/filters/channel_idle/idle_filter_state.cc +1 -2
  100. data/src/core/ext/filters/channel_idle/idle_filter_state.h +0 -1
  101. data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +48 -57
  102. data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h +22 -19
  103. data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +25 -29
  104. data/src/core/ext/filters/fault_injection/fault_injection_filter.h +8 -7
  105. data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc +2 -3
  106. data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h +7 -10
  107. data/src/core/ext/filters/gcp_authentication/gcp_authentication_filter.cc +205 -0
  108. data/src/core/ext/filters/gcp_authentication/gcp_authentication_filter.h +97 -0
  109. data/src/core/ext/filters/gcp_authentication/gcp_authentication_service_config_parser.cc +80 -0
  110. data/src/core/ext/filters/gcp_authentication/gcp_authentication_service_config_parser.h +86 -0
  111. data/src/core/ext/filters/http/client/http_client_filter.cc +19 -12
  112. data/src/core/ext/filters/http/client/http_client_filter.h +8 -6
  113. data/src/core/ext/filters/http/client_authority_filter.cc +13 -11
  114. data/src/core/ext/filters/http/client_authority_filter.h +8 -5
  115. data/src/core/ext/filters/http/http_filters_plugin.cc +16 -37
  116. data/src/core/ext/filters/http/message_compress/compression_filter.cc +67 -61
  117. data/src/core/ext/filters/http/message_compress/compression_filter.h +18 -13
  118. data/src/core/ext/filters/http/server/http_server_filter.cc +19 -16
  119. data/src/core/ext/filters/http/server/http_server_filter.h +8 -6
  120. data/src/core/ext/filters/message_size/message_size_filter.cc +51 -55
  121. data/src/core/ext/filters/message_size/message_size_filter.h +20 -16
  122. data/src/core/ext/filters/rbac/rbac_filter.cc +14 -19
  123. data/src/core/ext/filters/rbac/rbac_filter.h +8 -7
  124. data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +18 -12
  125. data/src/core/ext/filters/rbac/rbac_service_config_parser.h +3 -5
  126. data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +26 -28
  127. data/src/core/ext/filters/stateful_session/stateful_session_filter.h +8 -6
  128. data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc +2 -3
  129. data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +6 -8
  130. data/src/core/ext/transport/chttp2/alpn/alpn.cc +4 -5
  131. data/src/core/ext/transport/chttp2/alpn/alpn.h +0 -1
  132. data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +87 -124
  133. data/src/core/ext/transport/chttp2/client/chttp2_connector.h +5 -12
  134. data/src/core/ext/transport/chttp2/server/chttp2_server.cc +465 -316
  135. data/src/core/ext/transport/chttp2/server/chttp2_server.h +34 -1
  136. data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +26 -38
  137. data/src/core/ext/transport/chttp2/transport/bin_decoder.h +1 -3
  138. data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +10 -12
  139. data/src/core/ext/transport/chttp2/transport/bin_encoder.h +1 -3
  140. data/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc +53 -0
  141. data/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.h +72 -0
  142. data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +699 -622
  143. data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +37 -20
  144. data/src/core/ext/transport/chttp2/transport/context_list_entry.h +14 -12
  145. data/src/core/ext/transport/chttp2/transport/decode_huff.cc +2 -2
  146. data/src/core/ext/transport/chttp2/transport/decode_huff.h +2 -2
  147. data/src/core/ext/transport/chttp2/transport/flow_control.cc +11 -16
  148. data/src/core/ext/transport/chttp2/transport/flow_control.h +4 -9
  149. data/src/core/ext/transport/chttp2/transport/frame.cc +25 -8
  150. data/src/core/ext/transport/chttp2/transport/frame.h +11 -2
  151. data/src/core/ext/transport/chttp2/transport/frame_data.cc +21 -22
  152. data/src/core/ext/transport/chttp2/transport/frame_data.h +3 -5
  153. data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +7 -10
  154. data/src/core/ext/transport/chttp2/transport/frame_goaway.h +1 -3
  155. data/src/core/ext/transport/chttp2/transport/frame_ping.cc +16 -21
  156. data/src/core/ext/transport/chttp2/transport/frame_ping.h +1 -3
  157. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +19 -20
  158. data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +6 -6
  159. data/src/core/ext/transport/chttp2/transport/frame_security.cc +82 -0
  160. data/src/core/ext/transport/chttp2/transport/frame_security.h +44 -0
  161. data/src/core/ext/transport/chttp2/transport/frame_settings.cc +36 -21
  162. data/src/core/ext/transport/chttp2/transport/frame_settings.h +2 -4
  163. data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +31 -10
  164. data/src/core/ext/transport/chttp2/transport/frame_window_update.h +4 -4
  165. data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +21 -22
  166. data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +18 -15
  167. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +10 -10
  168. data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +3 -4
  169. data/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc +4 -5
  170. data/src/core/ext/transport/chttp2/transport/hpack_parse_result.h +5 -8
  171. data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +80 -123
  172. data/src/core/ext/transport/chttp2/transport/hpack_parser.h +5 -8
  173. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +45 -20
  174. data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +28 -9
  175. data/src/core/ext/transport/chttp2/transport/http2_settings.cc +12 -4
  176. data/src/core/ext/transport/chttp2/transport/http2_settings.h +10 -4
  177. data/src/core/ext/transport/chttp2/transport/huffsyms.cc +2 -2
  178. data/src/core/ext/transport/chttp2/transport/internal.h +73 -81
  179. data/src/core/ext/transport/chttp2/transport/legacy_frame.h +1 -0
  180. data/src/core/ext/transport/chttp2/transport/parsing.cc +140 -131
  181. data/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc +3 -4
  182. data/src/core/ext/transport/chttp2/transport/ping_abuse_policy.h +1 -1
  183. data/src/core/ext/transport/chttp2/transport/ping_callbacks.cc +4 -7
  184. data/src/core/ext/transport/chttp2/transport/ping_callbacks.h +2 -7
  185. data/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc +33 -19
  186. data/src/core/ext/transport/chttp2/transport/ping_rate_policy.h +6 -7
  187. data/src/core/ext/transport/chttp2/transport/stream_lists.cc +19 -21
  188. data/src/core/ext/transport/chttp2/transport/stream_lists.h +65 -0
  189. data/src/core/ext/transport/chttp2/transport/varint.cc +2 -2
  190. data/src/core/ext/transport/chttp2/transport/varint.h +2 -3
  191. data/src/core/ext/transport/chttp2/transport/write_size_policy.cc +4 -4
  192. data/src/core/ext/transport/chttp2/transport/write_size_policy.h +2 -3
  193. data/src/core/ext/transport/chttp2/transport/writing.cc +196 -155
  194. data/src/core/ext/transport/inproc/inproc_transport.cc +154 -70
  195. data/src/core/ext/transport/inproc/inproc_transport.h +2 -5
  196. data/src/core/ext/transport/inproc/legacy_inproc_transport.cc +136 -137
  197. data/src/core/ext/transport/inproc/legacy_inproc_transport.h +1 -4
  198. data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb.h +278 -234
  199. data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.c +73 -57
  200. data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.h +10 -4
  201. data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb.h +325 -269
  202. data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.c +87 -77
  203. data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.h +9 -4
  204. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb.h +277 -230
  205. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.c +75 -61
  206. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.h +10 -4
  207. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb.h +1074 -886
  208. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.c +299 -247
  209. data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.h +24 -4
  210. data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb.h +87 -80
  211. data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.c +22 -12
  212. data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.h +7 -4
  213. data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb.h +103 -88
  214. data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.c +28 -18
  215. data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.h +7 -4
  216. data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb.h +54 -46
  217. data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.c +20 -14
  218. data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.h +6 -4
  219. data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb.h +33 -28
  220. data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.c +14 -8
  221. data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.h +6 -4
  222. data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb.h +33 -28
  223. data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.c +14 -8
  224. data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.h +6 -4
  225. data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb.h +407 -321
  226. data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.c +104 -92
  227. data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.h +7 -4
  228. data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb.h +30 -24
  229. data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.c +14 -10
  230. data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.h +6 -4
  231. data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb.h +45 -36
  232. data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.c +11 -5
  233. data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.h +5 -4
  234. data/src/core/ext/upb-gen/envoy/annotations/resource.upb.h +38 -24
  235. data/src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.c +17 -10
  236. data/src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.h +6 -4
  237. data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb.h +578 -477
  238. data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.c +194 -148
  239. data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.h +21 -4
  240. data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb.h +1501 -1112
  241. data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.c +419 -331
  242. data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.h +29 -4
  243. data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb.h +194 -156
  244. data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.c +50 -50
  245. data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.h +8 -4
  246. data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb.h +1627 -1141
  247. data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.c +450 -395
  248. data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.h +31 -4
  249. data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb.h +41 -32
  250. data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.c +16 -12
  251. data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.h +6 -4
  252. data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb.h +360 -192
  253. data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.c +84 -73
  254. data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.h +6 -4
  255. data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb.h +660 -512
  256. data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.c +214 -160
  257. data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.h +20 -4
  258. data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb.h +384 -313
  259. data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.c +114 -94
  260. data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.h +13 -4
  261. data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb.h +34 -26
  262. data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.c +14 -14
  263. data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.h +6 -4
  264. data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb.h +1473 -692
  265. data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.c +450 -223
  266. data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.h +39 -4
  267. data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb.h +413 -347
  268. data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.c +113 -91
  269. data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.h +12 -4
  270. data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb.h +26 -20
  271. data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.c +13 -7
  272. data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.h +6 -4
  273. data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb.h +30 -24
  274. data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.c +14 -10
  275. data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.h +6 -4
  276. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb.h +87 -80
  277. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.c +22 -12
  278. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.h +7 -4
  279. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb.h +681 -474
  280. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.c +213 -162
  281. data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.h +19 -4
  282. data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb.h +808 -634
  283. data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.c +192 -150
  284. data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.h +14 -4
  285. data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb.h +56 -47
  286. data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.c +16 -10
  287. data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.h +6 -4
  288. data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb.h +42 -34
  289. data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.c +17 -11
  290. data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.h +6 -4
  291. data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb.h +872 -641
  292. data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.c +266 -236
  293. data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.h +21 -4
  294. data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb.h +72 -65
  295. data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.c +23 -15
  296. data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.h +7 -4
  297. data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb.h +77 -65
  298. data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.c +24 -16
  299. data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.h +7 -4
  300. data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb.h +101 -87
  301. data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.c +30 -20
  302. data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.h +7 -4
  303. data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb.h +118 -97
  304. data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.c +35 -25
  305. data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.h +7 -4
  306. data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb.h +34 -26
  307. data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.c +14 -14
  308. data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.h +6 -4
  309. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb.h +197 -149
  310. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.c +59 -49
  311. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.h +9 -4
  312. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb.h +390 -291
  313. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.c +118 -90
  314. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.h +12 -4
  315. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb.h +578 -313
  316. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.c +132 -78
  317. data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.h +12 -4
  318. data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb.h +23 -18
  319. data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.c +12 -10
  320. data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.h +6 -4
  321. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb.h +615 -482
  322. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.c +165 -132
  323. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.h +15 -4
  324. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb.h +561 -472
  325. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.c +134 -116
  326. data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.h +12 -4
  327. data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb.h +172 -82
  328. data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.c +33 -41
  329. data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.h +6 -4
  330. data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb.h +52 -40
  331. data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.c +21 -29
  332. data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.h +7 -4
  333. data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb.h +55 -44
  334. data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.c +22 -16
  335. data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.h +6 -4
  336. data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb.h +340 -283
  337. data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.c +107 -77
  338. data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.h +13 -4
  339. data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb.h +433 -362
  340. data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.c +127 -85
  341. data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.h +15 -4
  342. data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb.h +665 -494
  343. data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.c +195 -161
  344. data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.h +16 -4
  345. data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb.h +391 -331
  346. data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.c +69 -77
  347. data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.h +8 -4
  348. data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb.h +3886 -3149
  349. data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.c +1036 -851
  350. data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.h +63 -4
  351. data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb.h +114 -95
  352. data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.c +39 -25
  353. data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.h +8 -4
  354. data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb.h +541 -434
  355. data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.c +171 -135
  356. data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.h +17 -4
  357. data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb.h +134 -28
  358. data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.c +43 -12
  359. data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.h +8 -4
  360. data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb.h +30 -24
  361. data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.c +14 -10
  362. data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.h +6 -4
  363. data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb.h +51 -40
  364. data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.c +24 -18
  365. data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.h +7 -4
  366. data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb.h +65 -59
  367. data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.c +18 -16
  368. data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.h +6 -4
  369. data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb.h +157 -140
  370. data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.c +39 -33
  371. data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.h +6 -4
  372. data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb.h +85 -69
  373. data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.c +24 -18
  374. data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.h +6 -4
  375. data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb.h +23 -18
  376. data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.c +12 -10
  377. data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.h +6 -4
  378. data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb.h +78 -62
  379. data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.c +29 -29
  380. data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.h +7 -4
  381. data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb.h +5 -4
  382. data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.c +3 -1
  383. data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.h +5 -4
  384. data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb.h +77 -60
  385. data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.c +29 -25
  386. data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.h +7 -4
  387. data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb.h +65 -54
  388. data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.c +25 -19
  389. data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.h +6 -4
  390. data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb.h +1104 -836
  391. data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.c +359 -304
  392. data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.h +21 -4
  393. data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +40 -39
  394. data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.c +10 -4
  395. data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.h +6 -4
  396. data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb.h +119 -92
  397. data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.c +52 -30
  398. data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.h +10 -4
  399. data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb.h +248 -208
  400. data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.c +65 -67
  401. data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.h +8 -4
  402. data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb.h +444 -0
  403. data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.c +135 -0
  404. data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.h +38 -0
  405. data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +113 -62
  406. data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.c +36 -30
  407. data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.h +7 -4
  408. data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb.h +181 -157
  409. data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.c +43 -35
  410. data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.h +7 -4
  411. data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h +60 -48
  412. data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.c +27 -17
  413. data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.h +7 -4
  414. data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +1546 -1234
  415. data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.c +422 -351
  416. data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.h +26 -4
  417. data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h +23 -18
  418. data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.c +12 -10
  419. data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.h +6 -4
  420. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +78 -58
  421. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.c +22 -26
  422. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.h +6 -4
  423. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb.h +145 -112
  424. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.c +60 -46
  425. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.h +10 -4
  426. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb.h +19 -16
  427. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.c +10 -4
  428. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.h +6 -4
  429. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +81 -62
  430. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.c +29 -23
  431. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.h +6 -4
  432. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h +23 -18
  433. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.c +12 -10
  434. data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.h +6 -4
  435. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb.h +106 -0
  436. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.c +52 -0
  437. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.h +32 -0
  438. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb.h +5 -4
  439. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.c +3 -1
  440. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.h +5 -4
  441. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb.h +662 -498
  442. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.c +142 -122
  443. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.h +14 -4
  444. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +109 -84
  445. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.c +44 -36
  446. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.h +8 -4
  447. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +574 -461
  448. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.c +153 -137
  449. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.h +12 -4
  450. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +70 -59
  451. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.c +24 -16
  452. data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.h +7 -4
  453. data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb.h +248 -189
  454. data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.c +75 -73
  455. data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.h +9 -4
  456. data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb.h +12 -10
  457. data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.c +8 -2
  458. data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.h +6 -4
  459. data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb.h +852 -714
  460. data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.c +219 -171
  461. data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.h +19 -4
  462. data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb.h +116 -102
  463. data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.c +32 -22
  464. data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.h +7 -4
  465. data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb.h +355 -292
  466. data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.c +100 -78
  467. data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.h +10 -4
  468. data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb.h +37 -30
  469. data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.c +17 -11
  470. data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.h +6 -4
  471. data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb.h +91 -75
  472. data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.c +36 -18
  473. data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.h +9 -4
  474. data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb.h +33 -26
  475. data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.c +15 -9
  476. data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.h +6 -4
  477. data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb.h +75 -64
  478. data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.c +38 -16
  479. data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.h +10 -4
  480. data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb.h +89 -75
  481. data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.c +27 -17
  482. data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.h +7 -4
  483. data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb.h +56 -47
  484. data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.c +16 -10
  485. data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.h +6 -4
  486. data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb.h +35 -28
  487. data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.c +15 -9
  488. data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.h +6 -4
  489. data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb.h +26 -20
  490. data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.c +13 -7
  491. data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.h +6 -4
  492. data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb.h +76 -60
  493. data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.c +36 -26
  494. data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.h +8 -4
  495. data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb.h +19 -16
  496. data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.c +13 -3
  497. data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.h +7 -4
  498. data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb.h +142 -93
  499. data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.c +45 -24
  500. data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.h +7 -4
  501. data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb.h +75 -63
  502. data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.c +23 -13
  503. data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.h +7 -4
  504. data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb.h +156 -125
  505. data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.c +54 -36
  506. data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.h +9 -4
  507. data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb.h +153 -125
  508. data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.c +61 -31
  509. data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.h +12 -4
  510. data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb.h +158 -126
  511. data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.c +64 -42
  512. data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.h +10 -4
  513. data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb.h +58 -46
  514. data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.c +28 -14
  515. data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.h +8 -4
  516. data/src/core/ext/upb-gen/envoy/type/v3/http.upb.h +5 -4
  517. data/src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.c +3 -1
  518. data/src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.h +5 -4
  519. data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb.h +19 -16
  520. data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.c +10 -4
  521. data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.h +6 -4
  522. data/src/core/ext/upb-gen/envoy/type/v3/percent.upb.h +40 -34
  523. data/src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.c +19 -9
  524. data/src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.h +7 -4
  525. data/src/core/ext/upb-gen/envoy/type/v3/range.upb.h +68 -58
  526. data/src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.c +30 -16
  527. data/src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.h +8 -4
  528. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb.h +67 -54
  529. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.c +27 -17
  530. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.h +7 -4
  531. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb.h +5 -4
  532. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.c +3 -1
  533. data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.h +5 -4
  534. data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb.h +33 -28
  535. data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.c +14 -8
  536. data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.h +6 -4
  537. data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb.h +41 -32
  538. data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.c +16 -12
  539. data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.h +6 -4
  540. data/src/core/ext/upb-gen/google/api/annotations.upb.h +24 -12
  541. data/src/core/ext/upb-gen/google/api/annotations.upb_minitable.c +6 -3
  542. data/src/core/ext/upb-gen/google/api/annotations.upb_minitable.h +5 -4
  543. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb.h +684 -528
  544. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.c +203 -166
  545. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.h +18 -4
  546. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb.h +1075 -547
  547. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.c +296 -178
  548. data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.h +23 -4
  549. data/src/core/ext/upb-gen/google/api/http.upb.h +194 -166
  550. data/src/core/ext/upb-gen/google/api/http.upb_minitable.c +53 -39
  551. data/src/core/ext/upb-gen/google/api/http.upb_minitable.h +8 -4
  552. data/src/core/ext/upb-gen/google/api/httpbody.upb.h +59 -51
  553. data/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c +17 -11
  554. data/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.h +6 -4
  555. data/src/core/ext/upb-gen/google/protobuf/any.upb.h +27 -22
  556. data/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c +12 -6
  557. data/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.h +6 -4
  558. data/src/core/ext/upb-gen/google/protobuf/descriptor.upb.h +2875 -2375
  559. data/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c +660 -618
  560. data/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.h +56 -21
  561. data/src/core/ext/upb-gen/google/protobuf/duration.upb.h +27 -22
  562. data/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c +12 -6
  563. data/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.h +6 -4
  564. data/src/core/ext/upb-gen/google/protobuf/empty.upb.h +13 -10
  565. data/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c +8 -2
  566. data/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.h +6 -4
  567. data/src/core/ext/upb-gen/google/protobuf/struct.upb.h +161 -120
  568. data/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c +54 -38
  569. data/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.h +9 -4
  570. data/src/core/ext/upb-gen/google/protobuf/timestamp.upb.h +27 -22
  571. data/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c +12 -6
  572. data/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.h +6 -4
  573. data/src/core/ext/upb-gen/google/protobuf/wrappers.upb.h +132 -112
  574. data/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c +66 -28
  575. data/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.h +14 -4
  576. data/src/core/ext/upb-gen/google/rpc/status.upb.h +59 -51
  577. data/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c +17 -11
  578. data/src/core/ext/upb-gen/google/rpc/status.upb_minitable.h +6 -4
  579. data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb.h +119 -96
  580. data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.c +48 -30
  581. data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.h +9 -4
  582. data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb.h +90 -65
  583. data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c +34 -24
  584. data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.h +7 -4
  585. data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb.h +640 -522
  586. data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c +188 -144
  587. data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.h +17 -4
  588. data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb.h +56 -44
  589. data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c +23 -19
  590. data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.h +7 -4
  591. data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb.h +34 -28
  592. data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c +17 -7
  593. data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.h +7 -4
  594. data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb.h +294 -242
  595. data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c +106 -74
  596. data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.h +14 -4
  597. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb.h +108 -86
  598. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c +36 -22
  599. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.h +8 -4
  600. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb.h +553 -470
  601. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c +128 -94
  602. data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.h +14 -4
  603. data/src/core/ext/upb-gen/udpa/annotations/migrate.upb.h +149 -86
  604. data/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c +47 -28
  605. data/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.h +8 -4
  606. data/src/core/ext/upb-gen/udpa/annotations/security.upb.h +45 -30
  607. data/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c +19 -12
  608. data/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.h +6 -4
  609. data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb.h +15 -12
  610. data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c +5 -2
  611. data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.h +5 -4
  612. data/src/core/ext/upb-gen/udpa/annotations/status.upb.h +45 -30
  613. data/src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.c +19 -12
  614. data/src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.h +6 -4
  615. data/src/core/ext/upb-gen/udpa/annotations/versioning.upb.h +38 -24
  616. data/src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.c +17 -10
  617. data/src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.h +6 -4
  618. data/src/core/ext/upb-gen/validate/validate.upb.h +2716 -2462
  619. data/src/core/ext/upb-gen/validate/validate.upb_minitable.c +505 -551
  620. data/src/core/ext/upb-gen/validate/validate.upb_minitable.h +29 -5
  621. data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb.h +149 -86
  622. data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.c +47 -28
  623. data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.h +8 -4
  624. data/src/core/ext/upb-gen/xds/annotations/v3/security.upb.h +45 -30
  625. data/src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.c +19 -12
  626. data/src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.h +6 -4
  627. data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb.h +15 -12
  628. data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.c +5 -2
  629. data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.h +5 -4
  630. data/src/core/ext/upb-gen/xds/annotations/v3/status.upb.h +125 -82
  631. data/src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.c +49 -28
  632. data/src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.h +9 -4
  633. data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb.h +38 -24
  634. data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.c +17 -10
  635. data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.h +6 -4
  636. data/src/core/ext/upb-gen/xds/core/v3/authority.upb.h +19 -16
  637. data/src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.c +10 -4
  638. data/src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.h +6 -4
  639. data/src/core/ext/upb-gen/xds/core/v3/cidr.upb.h +30 -24
  640. data/src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.c +14 -10
  641. data/src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.h +6 -4
  642. data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb.h +69 -54
  643. data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.c +30 -20
  644. data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.h +7 -4
  645. data/src/core/ext/upb-gen/xds/core/v3/context_params.upb.h +44 -27
  646. data/src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.c +21 -11
  647. data/src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.h +7 -4
  648. data/src/core/ext/upb-gen/xds/core/v3/extension.upb.h +30 -24
  649. data/src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.c +14 -10
  650. data/src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.h +6 -4
  651. data/src/core/ext/upb-gen/xds/core/v3/resource.upb.h +41 -32
  652. data/src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.c +18 -12
  653. data/src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.h +6 -4
  654. data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb.h +117 -97
  655. data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.c +36 -26
  656. data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.h +7 -4
  657. data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb.h +44 -36
  658. data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.c +18 -16
  659. data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.h +6 -4
  660. data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb.h +150 -97
  661. data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c +55 -37
  662. data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.h +9 -4
  663. data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb.h +51 -47
  664. data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c +15 -9
  665. data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.h +6 -4
  666. data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb.h +30 -24
  667. data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.c +15 -9
  668. data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.h +6 -4
  669. data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb.h +91 -82
  670. data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.c +24 -16
  671. data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.h +7 -4
  672. data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb.h +12 -10
  673. data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.c +8 -2
  674. data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.h +6 -4
  675. data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb.h +103 -88
  676. data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.c +28 -18
  677. data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.h +7 -4
  678. data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb.h +387 -289
  679. data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.c +137 -103
  680. data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.h +15 -4
  681. data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb.h +278 -238
  682. data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.c +69 -49
  683. data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.h +11 -4
  684. data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb.h +40 -32
  685. data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.c +20 -10
  686. data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.h +7 -4
  687. data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb.h +109 -93
  688. data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.c +33 -23
  689. data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.h +7 -4
  690. data/src/core/ext/upb-gen/xds/type/v3/cel.upb.h +66 -50
  691. data/src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.c +27 -23
  692. data/src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.h +7 -4
  693. data/src/core/ext/upb-gen/xds/type/v3/range.upb.h +69 -58
  694. data/src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.c +30 -16
  695. data/src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.h +8 -4
  696. data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb.h +30 -24
  697. data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.c +14 -10
  698. data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.h +6 -4
  699. data/src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.c +2 -1
  700. data/src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.h +7 -9
  701. data/src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.c +2 -1
  702. data/src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.h +7 -9
  703. data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.c +2 -1
  704. data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.h +7 -9
  705. data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.c +2 -1
  706. data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.h +7 -9
  707. data/src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.c +2 -1
  708. data/src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.h +7 -9
  709. data/src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.c +2 -1
  710. data/src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.h +7 -9
  711. data/src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.c +2 -1
  712. data/src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.h +7 -9
  713. data/src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.c +2 -1
  714. data/src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.h +7 -9
  715. data/src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.c +2 -1
  716. data/src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.h +7 -9
  717. data/src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.c +88 -82
  718. data/src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.h +7 -9
  719. data/src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.c +2 -1
  720. data/src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.h +7 -9
  721. data/src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.c +2 -1
  722. data/src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.h +7 -9
  723. data/src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.c +2 -1
  724. data/src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.h +7 -9
  725. data/src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.c +63 -61
  726. data/src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.h +7 -9
  727. data/src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +223 -211
  728. data/src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.h +12 -9
  729. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.c +2 -1
  730. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.h +7 -9
  731. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.c +319 -298
  732. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.h +12 -9
  733. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.c +2 -1
  734. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.h +7 -9
  735. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.c +116 -106
  736. data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.h +7 -9
  737. data/src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.c +2 -1
  738. data/src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.h +7 -9
  739. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.c +2 -1
  740. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.h +7 -9
  741. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.c +2 -1
  742. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.h +7 -9
  743. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.c +187 -141
  744. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.h +32 -9
  745. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.c +12 -12
  746. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.h +7 -9
  747. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.c +2 -1
  748. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.h +7 -9
  749. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.c +2 -1
  750. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.h +7 -9
  751. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.c +2 -1
  752. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.h +7 -9
  753. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.c +175 -165
  754. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.h +7 -9
  755. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.c +199 -188
  756. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.h +7 -9
  757. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.c +2 -1
  758. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.h +7 -9
  759. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.c +2 -1
  760. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.h +7 -9
  761. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.c +231 -223
  762. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.h +7 -9
  763. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.c +2 -1
  764. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.h +7 -9
  765. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.c +2 -1
  766. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.h +7 -9
  767. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.c +2 -1
  768. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.h +7 -9
  769. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.c +2 -1
  770. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.h +7 -9
  771. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.c +2 -1
  772. data/src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.h +7 -9
  773. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.c +2 -1
  774. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.h +7 -9
  775. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.c +41 -37
  776. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.h +7 -9
  777. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.c +124 -94
  778. data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.h +12 -9
  779. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.c +2 -1
  780. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.h +7 -9
  781. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.c +43 -40
  782. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.h +7 -9
  783. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.c +2 -1
  784. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.h +7 -9
  785. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.c +22 -13
  786. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.h +7 -9
  787. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.c +2 -1
  788. data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.h +7 -9
  789. data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.c +2 -1
  790. data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.h +7 -9
  791. data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.c +2 -1
  792. data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.h +7 -9
  793. data/src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.c +2 -1
  794. data/src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.h +7 -9
  795. data/src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.c +71 -66
  796. data/src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.h +7 -9
  797. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.c +2 -1
  798. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.h +7 -9
  799. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.c +613 -605
  800. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.h +7 -9
  801. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.c +2 -1
  802. data/src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.h +7 -9
  803. data/src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.c +2 -1
  804. data/src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.h +7 -9
  805. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.c +32 -21
  806. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.h +12 -9
  807. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.c +20 -18
  808. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.h +7 -9
  809. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.c +2 -1
  810. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.h +7 -9
  811. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.c +2 -1
  812. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.h +7 -9
  813. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.c +64 -60
  814. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.h +7 -9
  815. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.c +2 -1
  816. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.h +7 -9
  817. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.c +2 -1
  818. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.h +7 -9
  819. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.c +2 -1
  820. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.h +7 -9
  821. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.c +2 -1
  822. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.h +7 -9
  823. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.c +2 -1
  824. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.h +7 -9
  825. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.c +23 -21
  826. data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.h +7 -9
  827. data/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c +147 -143
  828. data/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.h +7 -9
  829. data/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c +2 -1
  830. data/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.h +7 -9
  831. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c +2 -1
  832. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.h +7 -9
  833. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c +2 -1
  834. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.h +7 -9
  835. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.c +87 -0
  836. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.h +48 -0
  837. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c +38 -34
  838. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.h +7 -9
  839. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.c +2 -1
  840. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.h +7 -9
  841. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c +2 -1
  842. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h +7 -9
  843. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +292 -289
  844. data/src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h +7 -9
  845. data/src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c +2 -1
  846. data/src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h +7 -9
  847. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.c +53 -0
  848. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.h +33 -0
  849. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c +2 -1
  850. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.h +7 -9
  851. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c +84 -76
  852. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.h +12 -9
  853. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c +2 -1
  854. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.h +7 -9
  855. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +2 -1
  856. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h +7 -9
  857. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c +2 -1
  858. data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.h +7 -9
  859. data/src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.c +2 -1
  860. data/src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.h +7 -9
  861. data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.c +2 -1
  862. data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.h +7 -9
  863. data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.c +2 -1
  864. data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.h +7 -9
  865. data/src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.c +2 -1
  866. data/src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.h +7 -9
  867. data/src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.c +2 -1
  868. data/src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.h +7 -9
  869. data/src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.c +2 -1
  870. data/src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.h +7 -9
  871. data/src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.c +2 -1
  872. data/src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.h +7 -9
  873. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.c +2 -1
  874. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.h +7 -9
  875. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.c +2 -1
  876. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.h +7 -9
  877. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.c +2 -1
  878. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.h +7 -9
  879. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.c +2 -1
  880. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.h +7 -9
  881. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.c +2 -1
  882. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.h +7 -9
  883. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.c +2 -1
  884. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.h +7 -9
  885. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.c +2 -1
  886. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.h +7 -9
  887. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.c +2 -1
  888. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.h +7 -9
  889. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.c +41 -34
  890. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.h +7 -9
  891. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.c +2 -1
  892. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.h +7 -9
  893. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.c +2 -1
  894. data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.h +7 -9
  895. data/src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.c +2 -1
  896. data/src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.h +7 -9
  897. data/src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.c +2 -1
  898. data/src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.h +7 -9
  899. data/src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.c +2 -1
  900. data/src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.h +7 -9
  901. data/src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.c +2 -1
  902. data/src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.h +7 -9
  903. data/src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.c +2 -1
  904. data/src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.h +7 -9
  905. data/src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.c +2 -1
  906. data/src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.h +7 -9
  907. data/src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.c +2 -1
  908. data/src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.h +7 -9
  909. data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.c +2 -1
  910. data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.h +7 -9
  911. data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.c +2 -1
  912. data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.h +7 -9
  913. data/src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.c +2 -1
  914. data/src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.h +7 -9
  915. data/src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.c +2 -1
  916. data/src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.h +7 -9
  917. data/src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.c +2 -1
  918. data/src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.h +7 -9
  919. data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.c +110 -108
  920. data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.h +7 -9
  921. data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.c +103 -79
  922. data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.h +17 -9
  923. data/src/core/ext/upbdefs-gen/google/api/http.upbdefs.c +2 -1
  924. data/src/core/ext/upbdefs-gen/google/api/http.upbdefs.h +7 -9
  925. data/src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.c +2 -1
  926. data/src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.h +7 -9
  927. data/src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.c +2 -1
  928. data/src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.h +7 -9
  929. data/src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.c +478 -450
  930. data/src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.h +12 -9
  931. data/src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.c +2 -1
  932. data/src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.h +7 -9
  933. data/src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.c +2 -1
  934. data/src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.h +7 -9
  935. data/src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.c +2 -1
  936. data/src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.h +7 -9
  937. data/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c +2 -1
  938. data/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.h +7 -9
  939. data/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c +2 -1
  940. data/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.h +7 -9
  941. data/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c +2 -1
  942. data/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.h +7 -9
  943. data/src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.c +2 -1
  944. data/src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.h +7 -9
  945. data/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c +2 -1
  946. data/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.h +7 -9
  947. data/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c +2 -1
  948. data/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.h +7 -9
  949. data/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c +2 -1
  950. data/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.h +7 -9
  951. data/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c +2 -1
  952. data/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.h +7 -9
  953. data/src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.c +2 -1
  954. data/src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.h +7 -9
  955. data/src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.c +2 -1
  956. data/src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.h +7 -9
  957. data/src/core/ext/upbdefs-gen/validate/validate.upbdefs.c +2 -1
  958. data/src/core/ext/upbdefs-gen/validate/validate.upbdefs.h +7 -9
  959. data/src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.c +2 -1
  960. data/src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.h +7 -9
  961. data/src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.c +2 -1
  962. data/src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.h +7 -9
  963. data/src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.c +2 -1
  964. data/src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.h +7 -9
  965. data/src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.c +2 -1
  966. data/src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.h +7 -9
  967. data/src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.c +2 -1
  968. data/src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.h +7 -9
  969. data/src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.c +2 -1
  970. data/src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.h +7 -9
  971. data/src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.c +2 -1
  972. data/src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.h +7 -9
  973. data/src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.c +2 -1
  974. data/src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.h +7 -9
  975. data/src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.c +2 -1
  976. data/src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.h +7 -9
  977. data/src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.c +2 -1
  978. data/src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.h +7 -9
  979. data/src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.c +2 -1
  980. data/src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.h +7 -9
  981. data/src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.c +2 -1
  982. data/src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.h +7 -9
  983. data/src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.c +2 -1
  984. data/src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.h +7 -9
  985. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.c +2 -1
  986. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.h +7 -9
  987. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.c +2 -1
  988. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.h +7 -9
  989. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.c +2 -1
  990. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.h +7 -9
  991. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.c +2 -1
  992. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.h +7 -9
  993. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.c +2 -1
  994. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.h +7 -9
  995. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.c +2 -1
  996. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.h +7 -9
  997. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.c +2 -1
  998. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.h +7 -9
  999. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.c +2 -1
  1000. data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.h +7 -9
  1001. data/src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.c +2 -1
  1002. data/src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.h +7 -9
  1003. data/src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.c +2 -1
  1004. data/src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.h +7 -9
  1005. data/src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.c +2 -1
  1006. data/src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.h +7 -9
  1007. data/src/core/filter/blackboard.cc +33 -0
  1008. data/src/core/filter/blackboard.h +70 -0
  1009. data/src/core/handshaker/endpoint_info/endpoint_info_handshaker.cc +81 -0
  1010. data/src/core/handshaker/endpoint_info/endpoint_info_handshaker.h +37 -0
  1011. data/src/core/handshaker/handshaker.cc +199 -0
  1012. data/src/core/handshaker/handshaker.h +170 -0
  1013. data/src/core/handshaker/handshaker_factory.h +89 -0
  1014. data/src/core/handshaker/handshaker_registry.cc +60 -0
  1015. data/src/core/handshaker/handshaker_registry.h +69 -0
  1016. data/src/core/handshaker/http_connect/http_connect_handshaker.cc +359 -0
  1017. data/src/core/handshaker/http_connect/http_connect_handshaker.h +42 -0
  1018. data/src/core/handshaker/http_connect/http_proxy_mapper.cc +297 -0
  1019. data/src/core/handshaker/http_connect/http_proxy_mapper.h +52 -0
  1020. data/src/core/handshaker/http_connect/xds_http_proxy_mapper.cc +57 -0
  1021. data/src/core/handshaker/http_connect/xds_http_proxy_mapper.h +46 -0
  1022. data/src/core/handshaker/proxy_mapper.h +52 -0
  1023. data/src/core/handshaker/proxy_mapper_registry.cc +71 -0
  1024. data/src/core/handshaker/proxy_mapper_registry.h +74 -0
  1025. data/src/core/handshaker/security/secure_endpoint.cc +578 -0
  1026. data/src/core/handshaker/security/secure_endpoint.h +41 -0
  1027. data/src/core/handshaker/security/security_handshaker.cc +628 -0
  1028. data/src/core/handshaker/security/security_handshaker.h +45 -0
  1029. data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc +230 -0
  1030. data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.h +39 -0
  1031. data/src/core/lib/address_utils/parse_address.cc +41 -45
  1032. data/src/core/lib/address_utils/parse_address.h +1 -3
  1033. data/src/core/lib/address_utils/sockaddr_utils.cc +30 -20
  1034. data/src/core/lib/address_utils/sockaddr_utils.h +0 -2
  1035. data/src/core/lib/channel/call_finalization.h +1 -1
  1036. data/src/core/lib/channel/channel_args.cc +20 -26
  1037. data/src/core/lib/channel/channel_args.h +34 -21
  1038. data/src/core/lib/channel/channel_args_preconditioning.cc +2 -2
  1039. data/src/core/lib/channel/channel_args_preconditioning.h +2 -3
  1040. data/src/core/lib/channel/channel_stack.cc +21 -74
  1041. data/src/core/lib/channel/channel_stack.h +31 -62
  1042. data/src/core/lib/channel/channel_stack_builder.cc +2 -2
  1043. data/src/core/lib/channel/channel_stack_builder.h +1 -7
  1044. data/src/core/lib/channel/channel_stack_builder_impl.cc +18 -30
  1045. data/src/core/lib/channel/channel_stack_builder_impl.h +13 -3
  1046. data/src/core/lib/channel/connected_channel.cc +49 -709
  1047. data/src/core/lib/channel/promise_based_filter.cc +249 -324
  1048. data/src/core/lib/channel/promise_based_filter.h +269 -508
  1049. data/src/core/lib/channel/status_util.cc +2 -4
  1050. data/src/core/lib/channel/status_util.h +1 -2
  1051. data/src/core/lib/compression/compression.cc +7 -10
  1052. data/src/core/lib/compression/compression_internal.cc +39 -9
  1053. data/src/core/lib/compression/compression_internal.h +5 -5
  1054. data/src/core/lib/compression/message_compress.cc +16 -18
  1055. data/src/core/lib/compression/message_compress.h +1 -2
  1056. data/src/core/lib/debug/trace.cc +48 -67
  1057. data/src/core/lib/debug/trace.h +2 -97
  1058. data/src/core/lib/debug/trace_flags.cc +242 -0
  1059. data/src/core/lib/debug/trace_flags.h +131 -0
  1060. data/src/core/lib/debug/trace_impl.h +125 -0
  1061. data/src/core/lib/event_engine/ares_resolver.cc +165 -92
  1062. data/src/core/lib/event_engine/ares_resolver.h +11 -18
  1063. data/src/core/lib/event_engine/cf_engine/cf_engine.cc +14 -15
  1064. data/src/core/lib/event_engine/cf_engine/cf_engine.h +1 -1
  1065. data/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc +47 -34
  1066. data/src/core/lib/event_engine/cf_engine/cfstream_endpoint.h +4 -6
  1067. data/src/core/lib/event_engine/cf_engine/cftype_unique_ref.h +1 -1
  1068. data/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc +28 -31
  1069. data/src/core/lib/event_engine/cf_engine/dns_service_resolver.h +5 -6
  1070. data/src/core/lib/event_engine/channel_args_endpoint_config.cc +3 -5
  1071. data/src/core/lib/event_engine/channel_args_endpoint_config.h +1 -3
  1072. data/src/core/lib/event_engine/common_closures.h +1 -2
  1073. data/src/core/lib/event_engine/default_event_engine.cc +13 -16
  1074. data/src/core/lib/event_engine/default_event_engine.h +3 -10
  1075. data/src/core/lib/event_engine/default_event_engine_factory.cc +3 -4
  1076. data/src/core/lib/event_engine/default_event_engine_factory.h +1 -2
  1077. data/src/core/lib/event_engine/event_engine.cc +35 -5
  1078. data/src/core/lib/event_engine/event_engine_context.h +32 -0
  1079. data/src/core/lib/event_engine/extensions/chaotic_good_extension.h +64 -0
  1080. data/src/core/lib/event_engine/extensions/supports_fd.h +25 -2
  1081. data/src/core/lib/event_engine/extensions/tcp_trace.h +42 -0
  1082. data/src/core/lib/event_engine/forkable.cc +12 -13
  1083. data/src/core/lib/event_engine/forkable.h +0 -13
  1084. data/src/core/lib/event_engine/grpc_polled_fd.h +2 -4
  1085. data/src/core/lib/event_engine/handle_containers.h +1 -3
  1086. data/src/core/lib/event_engine/memory_allocator_factory.h +1 -3
  1087. data/src/core/lib/event_engine/nameser.h +1 -1
  1088. data/src/core/lib/event_engine/poller.h +1 -2
  1089. data/src/core/lib/event_engine/posix.h +9 -2
  1090. data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +28 -30
  1091. data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.h +6 -8
  1092. data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +27 -31
  1093. data/src/core/lib/event_engine/posix_engine/ev_poll_posix.h +2 -4
  1094. data/src/core/lib/event_engine/posix_engine/event_poller.h +1 -3
  1095. data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +2 -3
  1096. data/src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h +3 -6
  1097. data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +5 -6
  1098. data/src/core/lib/event_engine/posix_engine/internal_errqueue.h +0 -1
  1099. data/src/core/lib/event_engine/posix_engine/lockfree_event.cc +7 -9
  1100. data/src/core/lib/event_engine/posix_engine/lockfree_event.h +0 -1
  1101. data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc +3 -4
  1102. data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.h +0 -1
  1103. data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +96 -103
  1104. data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +22 -23
  1105. data/src/core/lib/event_engine/posix_engine/posix_engine.cc +84 -65
  1106. data/src/core/lib/event_engine/posix_engine/posix_engine.h +16 -13
  1107. data/src/core/lib/event_engine/posix_engine/posix_engine_closure.h +1 -2
  1108. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +28 -31
  1109. data/src/core/lib/event_engine/posix_engine/posix_engine_listener.h +13 -13
  1110. data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +28 -33
  1111. data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.h +1 -3
  1112. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +53 -40
  1113. data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +10 -10
  1114. data/src/core/lib/event_engine/posix_engine/timer.cc +5 -6
  1115. data/src/core/lib/event_engine/posix_engine/timer.h +4 -7
  1116. data/src/core/lib/event_engine/posix_engine/timer_heap.cc +2 -3
  1117. data/src/core/lib/event_engine/posix_engine/timer_manager.cc +16 -25
  1118. data/src/core/lib/event_engine/posix_engine/timer_manager.h +4 -7
  1119. data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +7 -10
  1120. data/src/core/lib/event_engine/posix_engine/traced_buffer_list.h +2 -5
  1121. data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc +3 -4
  1122. data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.h +0 -1
  1123. data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc +2 -3
  1124. data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h +0 -1
  1125. data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc +0 -1
  1126. data/src/core/lib/event_engine/query_extensions.h +3 -2
  1127. data/src/core/lib/event_engine/ref_counted_dns_resolver_interface.h +2 -5
  1128. data/src/core/lib/event_engine/resolved_address.cc +8 -7
  1129. data/src/core/lib/event_engine/resolved_address_internal.h +1 -2
  1130. data/src/core/lib/event_engine/shim.cc +3 -3
  1131. data/src/core/lib/event_engine/slice.cc +5 -7
  1132. data/src/core/lib/event_engine/slice_buffer.cc +2 -4
  1133. data/src/core/lib/event_engine/tcp_socket_utils.cc +29 -27
  1134. data/src/core/lib/event_engine/tcp_socket_utils.h +3 -4
  1135. data/src/core/lib/event_engine/thread_local.cc +2 -2
  1136. data/src/core/lib/event_engine/thread_local.h +1 -1
  1137. data/src/core/lib/event_engine/thread_pool/thread_count.cc +5 -8
  1138. data/src/core/lib/event_engine/thread_pool/thread_count.h +4 -6
  1139. data/src/core/lib/event_engine/thread_pool/thread_pool.h +1 -4
  1140. data/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc +1 -2
  1141. data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc +46 -53
  1142. data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h +9 -15
  1143. data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc +50 -23
  1144. data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h +7 -7
  1145. data/src/core/lib/event_engine/time_util.cc +3 -4
  1146. data/src/core/lib/event_engine/time_util.h +1 -3
  1147. data/src/core/lib/event_engine/utils.cc +19 -6
  1148. data/src/core/lib/event_engine/utils.h +9 -4
  1149. data/src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc +158 -132
  1150. data/src/core/lib/event_engine/windows/grpc_polled_fd_windows.h +4 -6
  1151. data/src/core/lib/event_engine/windows/iocp.cc +24 -22
  1152. data/src/core/lib/event_engine/windows/iocp.h +2 -3
  1153. data/src/core/lib/event_engine/windows/native_windows_dns_resolver.cc +4 -6
  1154. data/src/core/lib/event_engine/windows/win_socket.cc +51 -36
  1155. data/src/core/lib/event_engine/windows/win_socket.h +11 -10
  1156. data/src/core/lib/event_engine/windows/windows_endpoint.cc +58 -44
  1157. data/src/core/lib/event_engine/windows/windows_engine.cc +265 -123
  1158. data/src/core/lib/event_engine/windows/windows_engine.h +143 -33
  1159. data/src/core/lib/event_engine/windows/windows_listener.cc +83 -49
  1160. data/src/core/lib/event_engine/windows/windows_listener.h +18 -5
  1161. data/src/core/lib/event_engine/work_queue/basic_work_queue.cc +3 -3
  1162. data/src/core/lib/event_engine/work_queue/basic_work_queue.h +2 -5
  1163. data/src/core/lib/event_engine/work_queue/work_queue.h +1 -3
  1164. data/src/core/lib/experiments/config.cc +111 -66
  1165. data/src/core/lib/experiments/config.h +55 -2
  1166. data/src/core/lib/experiments/experiments.cc +308 -510
  1167. data/src/core/lib/experiments/experiments.h +148 -240
  1168. data/src/core/lib/iomgr/buffer_list.cc +7 -8
  1169. data/src/core/lib/iomgr/buffer_list.h +2 -4
  1170. data/src/core/lib/iomgr/call_combiner.cc +42 -68
  1171. data/src/core/lib/iomgr/call_combiner.h +12 -17
  1172. data/src/core/lib/iomgr/cfstream_handle.cc +14 -20
  1173. data/src/core/lib/iomgr/cfstream_handle.h +1 -1
  1174. data/src/core/lib/iomgr/closure.cc +2 -2
  1175. data/src/core/lib/iomgr/closure.h +17 -21
  1176. data/src/core/lib/iomgr/combiner.cc +38 -52
  1177. data/src/core/lib/iomgr/combiner.h +2 -6
  1178. data/src/core/lib/iomgr/endpoint.cc +1 -7
  1179. data/src/core/lib/iomgr/endpoint.h +4 -6
  1180. data/src/core/lib/iomgr/endpoint_cfstream.cc +45 -76
  1181. data/src/core/lib/iomgr/endpoint_pair_posix.cc +8 -10
  1182. data/src/core/lib/iomgr/endpoint_pair_windows.cc +16 -17
  1183. data/src/core/lib/iomgr/error.cc +24 -34
  1184. data/src/core/lib/iomgr/error.h +8 -11
  1185. data/src/core/lib/iomgr/error_cfstream.cc +1 -3
  1186. data/src/core/lib/iomgr/ev_apple.cc +16 -24
  1187. data/src/core/lib/iomgr/ev_epoll1_linux.cc +81 -126
  1188. data/src/core/lib/iomgr/ev_poll_posix.cc +62 -62
  1189. data/src/core/lib/iomgr/ev_posix.cc +68 -60
  1190. data/src/core/lib/iomgr/ev_posix.h +9 -12
  1191. data/src/core/lib/iomgr/event_engine_shims/closure.cc +11 -15
  1192. data/src/core/lib/iomgr/event_engine_shims/closure.h +1 -3
  1193. data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +35 -53
  1194. data/src/core/lib/iomgr/event_engine_shims/endpoint.h +1 -2
  1195. data/src/core/lib/iomgr/event_engine_shims/tcp_client.cc +10 -15
  1196. data/src/core/lib/iomgr/event_engine_shims/tcp_client.h +1 -2
  1197. data/src/core/lib/iomgr/exec_ctx.cc +14 -18
  1198. data/src/core/lib/iomgr/exec_ctx.h +33 -23
  1199. data/src/core/lib/iomgr/executor.cc +60 -59
  1200. data/src/core/lib/iomgr/executor.h +2 -2
  1201. data/src/core/lib/iomgr/fork_posix.cc +11 -15
  1202. data/src/core/lib/iomgr/fork_windows.cc +3 -2
  1203. data/src/core/lib/iomgr/internal_errqueue.cc +5 -6
  1204. data/src/core/lib/iomgr/iocp_windows.cc +16 -15
  1205. data/src/core/lib/iomgr/iocp_windows.h +0 -1
  1206. data/src/core/lib/iomgr/iomgr.cc +19 -26
  1207. data/src/core/lib/iomgr/iomgr.h +0 -1
  1208. data/src/core/lib/iomgr/iomgr_internal.cc +1 -2
  1209. data/src/core/lib/iomgr/iomgr_internal.h +0 -1
  1210. data/src/core/lib/iomgr/iomgr_posix.cc +2 -1
  1211. data/src/core/lib/iomgr/iomgr_windows.cc +7 -6
  1212. data/src/core/lib/iomgr/lockfree_event.cc +12 -21
  1213. data/src/core/lib/iomgr/lockfree_event.h +1 -2
  1214. data/src/core/lib/iomgr/nameser.h +1 -1
  1215. data/src/core/lib/iomgr/polling_entity.cc +17 -11
  1216. data/src/core/lib/iomgr/pollset.cc +2 -2
  1217. data/src/core/lib/iomgr/pollset.h +0 -3
  1218. data/src/core/lib/iomgr/pollset_set.cc +2 -2
  1219. data/src/core/lib/iomgr/pollset_set_windows.cc +0 -1
  1220. data/src/core/lib/iomgr/pollset_windows.cc +2 -6
  1221. data/src/core/lib/iomgr/pollset_windows.h +0 -1
  1222. data/src/core/lib/iomgr/port.h +5 -2
  1223. data/src/core/lib/iomgr/python_util.h +1 -2
  1224. data/src/core/lib/iomgr/resolve_address.cc +4 -7
  1225. data/src/core/lib/iomgr/resolve_address.h +3 -6
  1226. data/src/core/lib/iomgr/resolve_address_impl.h +0 -1
  1227. data/src/core/lib/iomgr/resolve_address_posix.cc +12 -21
  1228. data/src/core/lib/iomgr/resolve_address_windows.cc +9 -12
  1229. data/src/core/lib/iomgr/resolved_address.h +0 -1
  1230. data/src/core/lib/iomgr/sockaddr_utils_posix.cc +5 -6
  1231. data/src/core/lib/iomgr/socket_factory_posix.cc +1 -1
  1232. data/src/core/lib/iomgr/socket_factory_posix.h +1 -2
  1233. data/src/core/lib/iomgr/socket_mutator.cc +3 -5
  1234. data/src/core/lib/iomgr/socket_mutator.h +2 -4
  1235. data/src/core/lib/iomgr/socket_utils.h +0 -1
  1236. data/src/core/lib/iomgr/socket_utils_common_posix.cc +45 -52
  1237. data/src/core/lib/iomgr/socket_utils_linux.cc +1 -3
  1238. data/src/core/lib/iomgr/socket_utils_posix.cc +3 -6
  1239. data/src/core/lib/iomgr/socket_utils_posix.h +1 -2
  1240. data/src/core/lib/iomgr/socket_utils_windows.cc +1 -3
  1241. data/src/core/lib/iomgr/socket_windows.cc +11 -14
  1242. data/src/core/lib/iomgr/socket_windows.h +1 -2
  1243. data/src/core/lib/iomgr/tcp_client.cc +2 -2
  1244. data/src/core/lib/iomgr/tcp_client.h +1 -2
  1245. data/src/core/lib/iomgr/tcp_client_cfstream.cc +12 -20
  1246. data/src/core/lib/iomgr/tcp_client_posix.cc +24 -36
  1247. data/src/core/lib/iomgr/tcp_client_windows.cc +33 -17
  1248. data/src/core/lib/iomgr/tcp_posix.cc +167 -189
  1249. data/src/core/lib/iomgr/tcp_posix.h +1 -3
  1250. data/src/core/lib/iomgr/tcp_server.cc +2 -2
  1251. data/src/core/lib/iomgr/tcp_server.h +3 -4
  1252. data/src/core/lib/iomgr/tcp_server_posix.cc +152 -156
  1253. data/src/core/lib/iomgr/tcp_server_utils_posix.h +0 -1
  1254. data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +13 -17
  1255. data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +13 -14
  1256. data/src/core/lib/iomgr/tcp_server_windows.cc +94 -50
  1257. data/src/core/lib/iomgr/tcp_windows.cc +38 -66
  1258. data/src/core/lib/iomgr/timer.cc +2 -2
  1259. data/src/core/lib/iomgr/timer.h +2 -3
  1260. data/src/core/lib/iomgr/timer_generic.cc +73 -106
  1261. data/src/core/lib/iomgr/timer_generic.h +0 -1
  1262. data/src/core/lib/iomgr/timer_heap.cc +4 -6
  1263. data/src/core/lib/iomgr/timer_manager.cc +27 -44
  1264. data/src/core/lib/iomgr/timer_manager.h +0 -1
  1265. data/src/core/lib/iomgr/unix_sockets_posix.cc +18 -6
  1266. data/src/core/lib/iomgr/unix_sockets_posix.h +1 -3
  1267. data/src/core/lib/iomgr/unix_sockets_posix_noop.cc +2 -4
  1268. data/src/core/lib/iomgr/vsock.cc +5 -8
  1269. data/src/core/lib/iomgr/vsock.h +1 -3
  1270. data/src/core/lib/iomgr/wakeup_fd_eventfd.cc +2 -4
  1271. data/src/core/lib/iomgr/wakeup_fd_pipe.cc +6 -7
  1272. data/src/core/lib/promise/activity.cc +4 -5
  1273. data/src/core/lib/promise/activity.h +54 -22
  1274. data/src/core/lib/promise/all_ok.h +52 -9
  1275. data/src/core/lib/promise/arena_promise.h +1 -3
  1276. data/src/core/lib/promise/cancel_callback.h +34 -6
  1277. data/src/core/lib/promise/context.h +20 -11
  1278. data/src/core/lib/promise/detail/basic_seq.h +33 -20
  1279. data/src/core/lib/promise/detail/join_state.h +555 -760
  1280. data/src/core/lib/promise/detail/promise_factory.h +45 -29
  1281. data/src/core/lib/promise/detail/promise_like.h +49 -11
  1282. data/src/core/lib/promise/detail/seq_state.h +1314 -1977
  1283. data/src/core/lib/promise/detail/status.h +36 -14
  1284. data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h +1 -2
  1285. data/src/core/lib/promise/for_each.h +97 -39
  1286. data/src/core/lib/promise/if.h +21 -17
  1287. data/src/core/lib/promise/interceptor_list.h +22 -36
  1288. data/src/core/lib/promise/latch.h +25 -40
  1289. data/src/core/lib/promise/loop.h +23 -10
  1290. data/src/core/lib/promise/map.h +77 -9
  1291. data/src/core/lib/promise/observable.h +181 -0
  1292. data/src/core/lib/promise/party.cc +263 -125
  1293. data/src/core/lib/promise/party.h +187 -391
  1294. data/src/core/lib/promise/pipe.h +34 -57
  1295. data/src/core/lib/promise/poll.h +117 -39
  1296. data/src/core/lib/promise/promise.h +12 -6
  1297. data/src/core/lib/promise/race.h +10 -5
  1298. data/src/core/lib/promise/seq.h +57 -64
  1299. data/src/core/lib/promise/sleep.cc +5 -6
  1300. data/src/core/lib/promise/sleep.h +2 -4
  1301. data/src/core/lib/promise/status_flag.h +165 -51
  1302. data/src/core/lib/promise/try_join.h +37 -20
  1303. data/src/core/lib/promise/try_seq.h +113 -77
  1304. data/src/core/lib/resource_quota/api.cc +4 -7
  1305. data/src/core/lib/resource_quota/api.h +3 -5
  1306. data/src/core/lib/resource_quota/arena.cc +64 -87
  1307. data/src/core/lib/resource_quota/arena.h +145 -215
  1308. data/src/core/lib/resource_quota/connection_quota.cc +69 -0
  1309. data/src/core/lib/resource_quota/connection_quota.h +60 -0
  1310. data/src/core/lib/resource_quota/memory_quota.cc +51 -52
  1311. data/src/core/lib/resource_quota/memory_quota.h +27 -21
  1312. data/src/core/lib/resource_quota/periodic_update.cc +5 -5
  1313. data/src/core/lib/resource_quota/periodic_update.h +1 -3
  1314. data/src/core/lib/resource_quota/resource_quota.cc +2 -2
  1315. data/src/core/lib/resource_quota/resource_quota.h +6 -8
  1316. data/src/core/lib/resource_quota/thread_quota.cc +4 -4
  1317. data/src/core/lib/resource_quota/thread_quota.h +3 -4
  1318. data/src/core/lib/security/authorization/audit_logging.cc +10 -12
  1319. data/src/core/lib/security/authorization/audit_logging.h +3 -5
  1320. data/src/core/lib/security/authorization/authorization_engine.h +1 -1
  1321. data/src/core/lib/security/authorization/authorization_policy_provider.h +5 -7
  1322. data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +3 -4
  1323. data/src/core/lib/security/authorization/evaluate_args.cc +16 -20
  1324. data/src/core/lib/security/authorization/evaluate_args.h +4 -6
  1325. data/src/core/lib/security/authorization/grpc_authorization_engine.cc +4 -5
  1326. data/src/core/lib/security/authorization/grpc_authorization_engine.h +1 -3
  1327. data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +29 -41
  1328. data/src/core/lib/security/authorization/grpc_server_authz_filter.h +11 -11
  1329. data/src/core/lib/security/authorization/matchers.cc +5 -8
  1330. data/src/core/lib/security/authorization/matchers.h +1 -3
  1331. data/src/core/lib/security/authorization/rbac_policy.cc +2 -2
  1332. data/src/core/lib/security/authorization/rbac_policy.h +2 -5
  1333. data/src/core/lib/security/authorization/stdout_logger.cc +7 -7
  1334. data/src/core/lib/security/authorization/stdout_logger.h +2 -3
  1335. data/src/core/lib/security/certificate_provider/certificate_provider_factory.h +7 -8
  1336. data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +6 -6
  1337. data/src/core/lib/security/certificate_provider/certificate_provider_registry.h +0 -1
  1338. data/src/core/lib/security/context/security_context.cc +51 -49
  1339. data/src/core/lib/security/context/security_context.h +46 -17
  1340. data/src/core/lib/security/credentials/alts/alts_credentials.cc +6 -7
  1341. data/src/core/lib/security/credentials/alts/alts_credentials.h +11 -7
  1342. data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +4 -5
  1343. data/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +1 -2
  1344. data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +3 -5
  1345. data/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc +3 -5
  1346. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +5 -7
  1347. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +4 -5
  1348. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.h +2 -2
  1349. data/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc +1 -2
  1350. data/src/core/lib/security/credentials/call_creds_util.cc +8 -8
  1351. data/src/core/lib/security/credentials/call_creds_util.h +2 -2
  1352. data/src/core/lib/security/credentials/channel_creds_registry.h +5 -6
  1353. data/src/core/lib/security/credentials/channel_creds_registry_init.cc +14 -13
  1354. data/src/core/lib/security/credentials/composite/composite_credentials.cc +18 -21
  1355. data/src/core/lib/security/credentials/composite/composite_credentials.h +9 -8
  1356. data/src/core/lib/security/credentials/credentials.cc +20 -20
  1357. data/src/core/lib/security/credentials/credentials.h +13 -14
  1358. data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +336 -339
  1359. data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +58 -46
  1360. data/src/core/lib/security/credentials/external/aws_request_signer.cc +4 -5
  1361. data/src/core/lib/security/credentials/external/aws_request_signer.h +1 -1
  1362. data/src/core/lib/security/credentials/external/external_account_credentials.cc +410 -374
  1363. data/src/core/lib/security/credentials/external/external_account_credentials.h +127 -59
  1364. data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +90 -53
  1365. data/src/core/lib/security/credentials/external/file_external_account_credentials.h +30 -9
  1366. data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +103 -129
  1367. data/src/core/lib/security/credentials/external/url_external_account_credentials.h +20 -22
  1368. data/src/core/lib/security/credentials/fake/fake_credentials.cc +2 -4
  1369. data/src/core/lib/security/credentials/fake/fake_credentials.h +10 -9
  1370. data/src/core/lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.cc +195 -0
  1371. data/src/core/lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.h +90 -0
  1372. data/src/core/lib/security/credentials/google_default/credentials_generic.cc +4 -6
  1373. data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +60 -77
  1374. data/src/core/lib/security/credentials/google_default/google_default_credentials.h +9 -7
  1375. data/src/core/lib/security/credentials/iam/iam_credentials.cc +9 -14
  1376. data/src/core/lib/security/credentials/iam/iam_credentials.h +6 -5
  1377. data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +2 -2
  1378. data/src/core/lib/security/credentials/insecure/insecure_credentials.h +4 -4
  1379. data/src/core/lib/security/credentials/jwt/json_token.cc +35 -34
  1380. data/src/core/lib/security/credentials/jwt/json_token.h +2 -4
  1381. data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +24 -30
  1382. data/src/core/lib/security/credentials/jwt/jwt_credentials.h +9 -9
  1383. data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +90 -102
  1384. data/src/core/lib/security/credentials/jwt/jwt_verifier.h +3 -5
  1385. data/src/core/lib/security/credentials/local/local_credentials.cc +3 -4
  1386. data/src/core/lib/security/credentials/local/local_credentials.h +5 -5
  1387. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +202 -302
  1388. data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +48 -71
  1389. data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +22 -35
  1390. data/src/core/lib/security/credentials/plugin/plugin_credentials.h +11 -13
  1391. data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +47 -59
  1392. data/src/core/lib/security/credentials/ssl/ssl_credentials.h +8 -9
  1393. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc +28 -28
  1394. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h +2 -3
  1395. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc +0 -2
  1396. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +107 -46
  1397. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +15 -14
  1398. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +8 -12
  1399. data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +9 -10
  1400. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +27 -29
  1401. data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +2 -1
  1402. data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +23 -20
  1403. data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.h +13 -12
  1404. data/src/core/lib/security/credentials/tls/tls_credentials.cc +20 -24
  1405. data/src/core/lib/security/credentials/tls/tls_credentials.h +7 -5
  1406. data/src/core/lib/security/credentials/tls/tls_utils.cc +5 -7
  1407. data/src/core/lib/security/credentials/tls/tls_utils.h +1 -2
  1408. data/src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.cc +305 -0
  1409. data/src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.h +181 -0
  1410. data/src/core/lib/security/credentials/xds/xds_credentials.cc +21 -19
  1411. data/src/core/lib/security/credentials/xds/xds_credentials.h +7 -9
  1412. data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +35 -40
  1413. data/src/core/lib/security/security_connector/alts/alts_security_connector.h +3 -3
  1414. data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +28 -28
  1415. data/src/core/lib/security/security_connector/fake/fake_security_connector.h +3 -3
  1416. data/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc +8 -10
  1417. data/src/core/lib/security/security_connector/insecure/insecure_security_connector.h +5 -6
  1418. data/src/core/lib/security/security_connector/load_system_roots.h +4 -2
  1419. data/src/core/lib/security/security_connector/load_system_roots_fallback.cc +3 -2
  1420. data/src/core/lib/security/security_connector/load_system_roots_supported.cc +8 -10
  1421. data/src/core/lib/security/security_connector/load_system_roots_supported.h +1 -2
  1422. data/src/core/lib/security/security_connector/load_system_roots_windows.cc +85 -0
  1423. data/src/core/lib/security/security_connector/local/local_security_connector.cc +35 -35
  1424. data/src/core/lib/security/security_connector/local/local_security_connector.h +3 -3
  1425. data/src/core/lib/security/security_connector/security_connector.cc +11 -15
  1426. data/src/core/lib/security/security_connector/security_connector.h +8 -11
  1427. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +28 -32
  1428. data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +4 -5
  1429. data/src/core/lib/security/security_connector/ssl_utils.cc +42 -42
  1430. data/src/core/lib/security/security_connector/ssl_utils.h +6 -9
  1431. data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +49 -62
  1432. data/src/core/lib/security/security_connector/tls/tls_security_connector.h +6 -7
  1433. data/src/core/lib/security/transport/auth_filters.h +17 -38
  1434. data/src/core/lib/security/transport/client_auth_filter.cc +19 -27
  1435. data/src/core/lib/security/transport/server_auth_filter.cc +23 -32
  1436. data/src/core/lib/security/util/json_util.cc +4 -6
  1437. data/src/core/lib/security/util/json_util.h +1 -1
  1438. data/src/core/lib/slice/percent_encoding.cc +4 -6
  1439. data/src/core/lib/slice/slice.cc +44 -20
  1440. data/src/core/lib/slice/slice.h +8 -11
  1441. data/src/core/lib/slice/slice_buffer.cc +67 -30
  1442. data/src/core/lib/slice/slice_buffer.h +16 -5
  1443. data/src/core/lib/slice/slice_internal.h +11 -7
  1444. data/src/core/lib/slice/slice_refcount.h +7 -15
  1445. data/src/core/lib/slice/slice_string_helpers.cc +3 -3
  1446. data/src/core/lib/slice/slice_string_helpers.h +1 -3
  1447. data/src/core/lib/surface/byte_buffer.cc +2 -4
  1448. data/src/core/lib/surface/byte_buffer_reader.cc +5 -7
  1449. data/src/core/lib/surface/call.cc +205 -3997
  1450. data/src/core/lib/surface/call.h +177 -101
  1451. data/src/core/lib/surface/call_details.cc +5 -5
  1452. data/src/core/lib/surface/call_log_batch.cc +7 -11
  1453. data/src/core/lib/surface/call_test_only.h +2 -4
  1454. data/src/core/lib/surface/call_utils.cc +231 -0
  1455. data/src/core/lib/surface/call_utils.h +486 -0
  1456. data/src/core/lib/surface/channel.cc +131 -340
  1457. data/src/core/lib/surface/channel.h +100 -118
  1458. data/src/core/lib/surface/channel_create.cc +125 -0
  1459. data/src/core/lib/surface/channel_create.h +42 -0
  1460. data/src/core/lib/surface/channel_init.cc +275 -271
  1461. data/src/core/lib/surface/channel_init.h +190 -115
  1462. data/src/core/lib/surface/channel_stack_type.cc +2 -2
  1463. data/src/core/lib/surface/client_call.cc +441 -0
  1464. data/src/core/lib/surface/client_call.h +187 -0
  1465. data/src/core/lib/surface/completion_queue.cc +127 -124
  1466. data/src/core/lib/surface/completion_queue.h +4 -14
  1467. data/src/core/lib/surface/completion_queue_factory.cc +8 -9
  1468. data/src/core/lib/surface/completion_queue_factory.h +1 -2
  1469. data/src/core/lib/surface/connection_context.cc +77 -0
  1470. data/src/core/lib/surface/connection_context.h +156 -0
  1471. data/src/core/lib/surface/event_string.cc +2 -2
  1472. data/src/core/lib/surface/event_string.h +1 -2
  1473. data/src/core/lib/surface/filter_stack_call.cc +1156 -0
  1474. data/src/core/lib/surface/filter_stack_call.h +366 -0
  1475. data/src/core/lib/surface/init.cc +66 -32
  1476. data/src/core/lib/surface/init.h +5 -0
  1477. data/src/core/lib/surface/init_internally.cc +2 -2
  1478. data/src/core/lib/surface/lame_client.cc +23 -54
  1479. data/src/core/lib/surface/lame_client.h +16 -15
  1480. data/src/core/lib/surface/legacy_channel.cc +428 -0
  1481. data/src/core/lib/surface/legacy_channel.h +113 -0
  1482. data/src/core/lib/surface/metadata_array.cc +6 -7
  1483. data/src/core/lib/surface/server_call.cc +227 -0
  1484. data/src/core/lib/surface/server_call.h +167 -0
  1485. data/src/core/lib/surface/validate_metadata.cc +4 -6
  1486. data/src/core/lib/surface/validate_metadata.h +3 -6
  1487. data/src/core/lib/surface/version.cc +3 -4
  1488. data/src/core/lib/transport/bdp_estimator.cc +14 -20
  1489. data/src/core/lib/transport/bdp_estimator.h +12 -18
  1490. data/src/core/lib/transport/call_arena_allocator.cc +27 -0
  1491. data/src/core/lib/transport/call_arena_allocator.h +91 -0
  1492. data/src/core/lib/transport/call_destination.h +76 -0
  1493. data/src/core/lib/transport/call_filters.cc +79 -308
  1494. data/src/core/lib/transport/call_filters.h +875 -476
  1495. data/src/core/lib/transport/call_final_info.cc +2 -2
  1496. data/src/core/lib/transport/call_final_info.h +3 -4
  1497. data/src/core/lib/transport/call_spine.cc +99 -79
  1498. data/src/core/lib/transport/call_spine.h +254 -248
  1499. data/src/core/lib/transport/call_state.cc +39 -0
  1500. data/src/core/lib/transport/call_state.h +1061 -0
  1501. data/src/core/lib/transport/connectivity_state.cc +38 -47
  1502. data/src/core/lib/transport/connectivity_state.h +3 -7
  1503. data/src/core/lib/transport/error_utils.cc +3 -5
  1504. data/src/core/lib/transport/error_utils.h +2 -4
  1505. data/src/core/lib/transport/interception_chain.cc +147 -0
  1506. data/src/core/lib/transport/interception_chain.h +253 -0
  1507. data/src/core/lib/transport/message.cc +3 -4
  1508. data/src/core/lib/transport/message.h +5 -0
  1509. data/src/core/lib/transport/metadata.cc +29 -5
  1510. data/src/core/lib/transport/metadata.h +37 -2
  1511. data/src/core/lib/transport/metadata_batch.cc +76 -8
  1512. data/src/core/lib/transport/metadata_batch.h +116 -31
  1513. data/src/core/lib/transport/metadata_compression_traits.h +2 -3
  1514. data/src/core/lib/transport/metadata_info.cc +55 -0
  1515. data/src/core/lib/transport/metadata_info.h +85 -0
  1516. data/src/core/lib/transport/parsed_metadata.cc +2 -2
  1517. data/src/core/lib/transport/parsed_metadata.h +2 -5
  1518. data/src/core/lib/transport/simple_slice_based_metadata.h +0 -1
  1519. data/src/core/lib/transport/status_conversion.cc +2 -2
  1520. data/src/core/lib/transport/status_conversion.h +2 -3
  1521. data/src/core/lib/transport/timeout_encoding.cc +11 -12
  1522. data/src/core/lib/transport/timeout_encoding.h +1 -3
  1523. data/src/core/lib/transport/transport.cc +7 -13
  1524. data/src/core/lib/transport/transport.h +94 -88
  1525. data/src/core/lib/transport/transport_framing_endpoint_extension.h +47 -0
  1526. data/src/core/lib/transport/transport_op_string.cc +2 -13
  1527. data/src/core/load_balancing/address_filtering.cc +3 -5
  1528. data/src/core/load_balancing/address_filtering.h +3 -4
  1529. data/src/core/load_balancing/backend_metric_parser.cc +94 -0
  1530. data/src/core/load_balancing/backend_metric_parser.h +45 -0
  1531. data/src/core/load_balancing/child_policy_handler.cc +43 -47
  1532. data/src/core/load_balancing/child_policy_handler.h +2 -3
  1533. data/src/core/load_balancing/delegating_helper.h +14 -8
  1534. data/src/core/load_balancing/endpoint_list.cc +18 -22
  1535. data/src/core/load_balancing/endpoint_list.h +20 -16
  1536. data/src/core/load_balancing/grpclb/client_load_reporting_filter.cc +38 -34
  1537. data/src/core/load_balancing/grpclb/client_load_reporting_filter.h +23 -7
  1538. data/src/core/load_balancing/grpclb/grpclb.cc +216 -255
  1539. data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc +2 -3
  1540. data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.h +1 -2
  1541. data/src/core/load_balancing/grpclb/grpclb_client_stats.cc +3 -5
  1542. data/src/core/load_balancing/grpclb/grpclb_client_stats.h +5 -8
  1543. data/src/core/load_balancing/grpclb/load_balancer_api.cc +7 -11
  1544. data/src/core/load_balancing/grpclb/load_balancer_api.h +3 -6
  1545. data/src/core/load_balancing/health_check_client.cc +68 -89
  1546. data/src/core/load_balancing/health_check_client.h +1 -1
  1547. data/src/core/load_balancing/health_check_client_internal.h +10 -13
  1548. data/src/core/load_balancing/lb_policy.cc +7 -10
  1549. data/src/core/load_balancing/lb_policy.h +84 -42
  1550. data/src/core/load_balancing/lb_policy_factory.h +3 -4
  1551. data/src/core/load_balancing/lb_policy_registry.cc +7 -9
  1552. data/src/core/load_balancing/lb_policy_registry.h +5 -6
  1553. data/src/core/load_balancing/oob_backend_metric.cc +28 -35
  1554. data/src/core/load_balancing/oob_backend_metric.h +1 -1
  1555. data/src/core/load_balancing/oob_backend_metric_internal.h +12 -14
  1556. data/src/core/load_balancing/outlier_detection/outlier_detection.cc +172 -217
  1557. data/src/core/load_balancing/outlier_detection/outlier_detection.h +5 -7
  1558. data/src/core/load_balancing/pick_first/pick_first.cc +1235 -409
  1559. data/src/core/load_balancing/priority/priority.cc +115 -153
  1560. data/src/core/load_balancing/ring_hash/ring_hash.cc +94 -101
  1561. data/src/core/load_balancing/ring_hash/ring_hash.h +6 -7
  1562. data/src/core/load_balancing/rls/rls.cc +516 -391
  1563. data/src/core/load_balancing/rls/rls.h +26 -0
  1564. data/src/core/load_balancing/round_robin/round_robin.cc +88 -559
  1565. data/src/core/load_balancing/subchannel_interface.h +18 -7
  1566. data/src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc +7 -8
  1567. data/src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h +2 -3
  1568. data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +240 -1042
  1569. data/src/core/load_balancing/weighted_target/weighted_target.cc +91 -124
  1570. data/src/core/load_balancing/weighted_target/weighted_target.h +28 -0
  1571. data/src/core/load_balancing/xds/cds.cc +70 -87
  1572. data/src/core/load_balancing/xds/xds_cluster_impl.cc +257 -188
  1573. data/src/core/load_balancing/xds/xds_cluster_manager.cc +59 -92
  1574. data/src/core/load_balancing/xds/xds_override_host.cc +141 -203
  1575. data/src/core/load_balancing/xds/xds_override_host.h +6 -7
  1576. data/src/core/load_balancing/xds/xds_wrr_locality.cc +45 -55
  1577. data/src/core/plugin_registry/grpc_plugin_registry.cc +28 -16
  1578. data/src/core/plugin_registry/grpc_plugin_registry_extra.cc +7 -4
  1579. data/src/core/resolver/dns/c_ares/dns_resolver_ares.cc +97 -72
  1580. data/src/core/resolver/dns/c_ares/dns_resolver_ares.h +1 -2
  1581. data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver.h +3 -5
  1582. data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +8 -11
  1583. data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +156 -137
  1584. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc +177 -162
  1585. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.h +3 -18
  1586. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +1 -1
  1587. data/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +2 -2
  1588. data/src/core/resolver/dns/dns_resolver_plugin.cc +11 -13
  1589. data/src/core/resolver/dns/dns_resolver_plugin.h +1 -1
  1590. data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc +58 -65
  1591. data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.h +5 -4
  1592. data/src/core/resolver/dns/event_engine/service_config_helper.cc +8 -10
  1593. data/src/core/resolver/dns/native/dns_resolver.cc +24 -34
  1594. data/src/core/resolver/dns/native/dns_resolver.h +1 -1
  1595. data/src/core/resolver/endpoint_addresses.cc +5 -8
  1596. data/src/core/resolver/endpoint_addresses.h +7 -5
  1597. data/src/core/resolver/fake/fake_resolver.cc +12 -14
  1598. data/src/core/resolver/fake/fake_resolver.h +7 -9
  1599. data/src/core/resolver/google_c2p/google_c2p_resolver.cc +38 -35
  1600. data/src/core/resolver/polling_resolver.cc +42 -58
  1601. data/src/core/resolver/polling_resolver.h +6 -8
  1602. data/src/core/resolver/resolver.cc +3 -7
  1603. data/src/core/resolver/resolver.h +2 -8
  1604. data/src/core/resolver/resolver_factory.h +2 -3
  1605. data/src/core/resolver/resolver_registry.cc +12 -14
  1606. data/src/core/resolver/resolver_registry.h +4 -5
  1607. data/src/core/resolver/sockaddr/sockaddr_resolver.cc +12 -14
  1608. data/src/core/resolver/xds/xds_config.cc +95 -0
  1609. data/src/core/resolver/xds/xds_config.h +108 -0
  1610. data/src/core/resolver/xds/xds_dependency_manager.cc +100 -183
  1611. data/src/core/resolver/xds/xds_dependency_manager.h +16 -81
  1612. data/src/core/resolver/xds/xds_resolver.cc +157 -142
  1613. data/src/core/resolver/xds/xds_resolver_attributes.h +7 -4
  1614. data/src/core/server/server.cc +1931 -0
  1615. data/src/core/server/server.h +569 -0
  1616. data/src/core/server/server_call_tracer_filter.cc +117 -0
  1617. data/src/core/server/server_call_tracer_filter.h +28 -0
  1618. data/src/core/server/server_config_selector.h +82 -0
  1619. data/src/core/server/server_config_selector_filter.cc +175 -0
  1620. data/src/core/server/server_config_selector_filter.h +33 -0
  1621. data/src/core/server/server_interface.h +45 -0
  1622. data/src/core/server/xds_channel_stack_modifier.cc +107 -0
  1623. data/src/core/server/xds_channel_stack_modifier.h +62 -0
  1624. data/src/core/server/xds_server_config_fetcher.cc +1384 -0
  1625. data/src/core/service_config/service_config.h +3 -6
  1626. data/src/core/service_config/service_config_call_data.h +20 -19
  1627. data/src/core/service_config/service_config_channel_arg_filter.cc +139 -0
  1628. data/src/core/service_config/service_config_impl.cc +9 -11
  1629. data/src/core/service_config/service_config_impl.h +6 -9
  1630. data/src/core/service_config/service_config_parser.cc +4 -9
  1631. data/src/core/service_config/service_config_parser.h +3 -5
  1632. data/src/core/telemetry/call_tracer.cc +386 -0
  1633. data/src/core/telemetry/call_tracer.h +263 -0
  1634. data/src/core/telemetry/histogram_view.cc +69 -0
  1635. data/src/core/telemetry/histogram_view.h +36 -0
  1636. data/src/core/telemetry/metrics.cc +180 -0
  1637. data/src/core/telemetry/metrics.h +569 -0
  1638. data/src/core/telemetry/stats.cc +67 -0
  1639. data/src/core/telemetry/stats.h +62 -0
  1640. data/src/core/telemetry/stats_data.cc +931 -0
  1641. data/src/core/telemetry/stats_data.h +705 -0
  1642. data/src/core/telemetry/tcp_tracer.h +143 -0
  1643. data/src/core/tsi/alts/crypt/aes_gcm.cc +4 -8
  1644. data/src/core/tsi/alts/crypt/gsec.cc +2 -4
  1645. data/src/core/tsi/alts/crypt/gsec.h +3 -5
  1646. data/src/core/tsi/alts/frame_protector/alts_counter.cc +2 -4
  1647. data/src/core/tsi/alts/frame_protector/alts_counter.h +2 -4
  1648. data/src/core/tsi/alts/frame_protector/alts_crypter.cc +2 -4
  1649. data/src/core/tsi/alts/frame_protector/alts_crypter.h +1 -3
  1650. data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +16 -19
  1651. data/src/core/tsi/alts/frame_protector/alts_frame_protector.h +0 -1
  1652. data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc +1 -2
  1653. data/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h +1 -2
  1654. data/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc +2 -3
  1655. data/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc +2 -3
  1656. data/src/core/tsi/alts/frame_protector/frame_handler.cc +11 -13
  1657. data/src/core/tsi/alts/frame_protector/frame_handler.h +0 -1
  1658. data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +84 -86
  1659. data/src/core/tsi/alts/handshaker/alts_handshaker_client.h +1 -2
  1660. data/src/core/tsi/alts/handshaker/alts_shared_resource.cc +5 -6
  1661. data/src/core/tsi/alts/handshaker/alts_shared_resource.h +2 -3
  1662. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +77 -76
  1663. data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +1 -2
  1664. data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +6 -5
  1665. data/src/core/tsi/alts/handshaker/alts_tsi_utils.h +1 -2
  1666. data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +17 -24
  1667. data/src/core/tsi/alts/handshaker/transport_security_common_api.h +4 -6
  1668. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +17 -19
  1669. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h +0 -1
  1670. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +12 -14
  1671. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h +0 -1
  1672. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol.h +1 -2
  1673. data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +16 -14
  1674. data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc +3 -6
  1675. data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h +0 -1
  1676. data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +13 -16
  1677. data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h +0 -1
  1678. data/src/core/tsi/fake_transport_security.cc +31 -34
  1679. data/src/core/tsi/local_transport_security.cc +9 -11
  1680. data/src/core/tsi/local_transport_security.h +1 -2
  1681. data/src/core/tsi/ssl/key_logging/ssl_key_logging.cc +14 -16
  1682. data/src/core/tsi/ssl/key_logging/ssl_key_logging.h +7 -10
  1683. data/src/core/tsi/ssl/session_cache/ssl_session.h +3 -5
  1684. data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +16 -17
  1685. data/src/core/tsi/ssl/session_cache/ssl_session_cache.h +8 -10
  1686. data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +4 -7
  1687. data/src/core/tsi/ssl_transport_security.cc +344 -193
  1688. data/src/core/tsi/ssl_transport_security.h +5 -8
  1689. data/src/core/tsi/ssl_transport_security_utils.cc +208 -27
  1690. data/src/core/tsi/ssl_transport_security_utils.h +40 -5
  1691. data/src/core/tsi/ssl_types.h +0 -1
  1692. data/src/core/tsi/transport_security.cc +3 -9
  1693. data/src/core/tsi/transport_security.h +0 -3
  1694. data/src/core/tsi/transport_security_grpc.cc +2 -2
  1695. data/src/core/tsi/transport_security_grpc.h +1 -2
  1696. data/src/core/tsi/transport_security_interface.h +2 -7
  1697. data/src/core/util/alloc.cc +69 -0
  1698. data/src/core/util/alloc.h +28 -0
  1699. data/src/core/util/atm.cc +34 -0
  1700. data/src/core/util/atomic_utils.h +47 -0
  1701. data/src/core/util/avl.h +324 -0
  1702. data/src/core/util/backoff.cc +46 -0
  1703. data/src/core/util/backoff.h +86 -0
  1704. data/src/core/util/bitset.h +224 -0
  1705. data/src/core/util/chunked_vector.h +256 -0
  1706. data/src/core/util/construct_destruct.h +41 -0
  1707. data/src/core/util/cpp_impl_of.h +49 -0
  1708. data/src/core/util/crash.cc +40 -0
  1709. data/src/core/util/crash.h +36 -0
  1710. data/src/core/util/debug_location.h +106 -0
  1711. data/src/core/util/directory_reader.h +48 -0
  1712. data/src/core/util/down_cast.h +48 -0
  1713. data/src/core/util/dual_ref_counted.h +376 -0
  1714. data/src/core/util/dump_args.cc +54 -0
  1715. data/src/core/util/dump_args.h +122 -0
  1716. data/src/core/util/env.h +53 -0
  1717. data/src/core/util/event_log.cc +87 -0
  1718. data/src/core/util/event_log.h +79 -0
  1719. data/src/core/util/examine_stack.cc +43 -0
  1720. data/src/core/util/examine_stack.h +45 -0
  1721. data/src/core/util/fork.cc +241 -0
  1722. data/src/core/util/fork.h +98 -0
  1723. data/src/core/util/gcp_metadata_query.cc +132 -0
  1724. data/src/core/util/gcp_metadata_query.h +86 -0
  1725. data/src/core/util/gethostname.h +26 -0
  1726. data/src/core/util/gethostname_fallback.cc +30 -0
  1727. data/src/core/util/gethostname_host_name_max.cc +39 -0
  1728. data/src/core/util/gethostname_sysconf.cc +39 -0
  1729. data/src/core/util/glob.cc +70 -0
  1730. data/src/core/util/glob.h +29 -0
  1731. data/src/core/util/gpr_time.cc +268 -0
  1732. data/src/core/util/grpc_if_nametoindex.h +29 -0
  1733. data/src/core/util/grpc_if_nametoindex_posix.cc +41 -0
  1734. data/src/core/util/grpc_if_nametoindex_unsupported.cc +36 -0
  1735. data/src/core/util/host_port.cc +114 -0
  1736. data/src/core/util/host_port.h +57 -0
  1737. data/src/core/util/http_client/format_request.cc +134 -0
  1738. data/src/core/util/http_client/format_request.h +37 -0
  1739. data/src/core/util/http_client/httpcli.cc +380 -0
  1740. data/src/core/util/http_client/httpcli.h +269 -0
  1741. data/src/core/util/http_client/httpcli_security_connector.cc +210 -0
  1742. data/src/core/util/http_client/httpcli_ssl_credentials.h +38 -0
  1743. data/src/core/util/http_client/parser.cc +447 -0
  1744. data/src/core/util/http_client/parser.h +126 -0
  1745. data/src/core/util/if_list.h +4530 -0
  1746. data/src/core/util/iphone/cpu.cc +43 -0
  1747. data/src/core/util/json/json.h +29 -0
  1748. data/src/core/util/json/json_args.h +34 -0
  1749. data/src/core/util/json/json_channel_args.h +41 -0
  1750. data/src/core/util/json/json_object_loader.cc +215 -0
  1751. data/src/core/util/json/json_object_loader.h +644 -0
  1752. data/src/core/util/json/json_reader.cc +953 -0
  1753. data/src/core/util/json/json_reader.h +33 -0
  1754. data/src/core/util/json/json_util.cc +101 -0
  1755. data/src/core/util/json/json_util.h +163 -0
  1756. data/src/core/util/json/json_writer.cc +337 -0
  1757. data/src/core/util/json/json_writer.h +33 -0
  1758. data/src/core/util/latent_see.cc +163 -0
  1759. data/src/core/util/latent_see.h +334 -0
  1760. data/src/core/util/linux/cpu.cc +86 -0
  1761. data/src/core/util/linux/env.cc +62 -0
  1762. data/src/core/util/load_file.cc +75 -0
  1763. data/src/core/util/load_file.h +33 -0
  1764. data/src/core/util/log.cc +127 -0
  1765. data/src/core/util/lru_cache.h +122 -0
  1766. data/src/core/util/manual_constructor.h +145 -0
  1767. data/src/core/util/match.h +74 -0
  1768. data/src/core/util/matchers.cc +337 -0
  1769. data/src/core/util/matchers.h +167 -0
  1770. data/src/core/util/memory.h +52 -0
  1771. data/src/core/util/mpscq.cc +108 -0
  1772. data/src/core/util/mpscq.h +98 -0
  1773. data/src/core/util/msys/tmpfile.cc +57 -0
  1774. data/src/core/util/no_destruct.h +95 -0
  1775. data/src/core/util/notification.h +66 -0
  1776. data/src/core/util/orphanable.h +153 -0
  1777. data/src/core/util/overload.h +59 -0
  1778. data/src/core/util/packed_table.h +40 -0
  1779. data/src/core/util/per_cpu.cc +34 -0
  1780. data/src/core/util/per_cpu.h +102 -0
  1781. data/src/core/util/posix/cpu.cc +83 -0
  1782. data/src/core/util/posix/directory_reader.cc +82 -0
  1783. data/src/core/util/posix/env.cc +47 -0
  1784. data/src/core/util/posix/stat.cc +54 -0
  1785. data/src/core/util/posix/string.cc +71 -0
  1786. data/src/core/util/posix/sync.cc +158 -0
  1787. data/src/core/util/posix/thd.cc +243 -0
  1788. data/src/core/util/posix/time.cc +123 -0
  1789. data/src/core/util/posix/tmpfile.cc +71 -0
  1790. data/src/core/util/random_early_detection.cc +33 -0
  1791. data/src/core/util/random_early_detection.h +61 -0
  1792. data/src/core/util/ref_counted.h +402 -0
  1793. data/src/core/util/ref_counted_ptr.h +443 -0
  1794. data/src/core/util/ref_counted_string.cc +42 -0
  1795. data/src/core/util/ref_counted_string.h +159 -0
  1796. data/src/core/util/ring_buffer.h +123 -0
  1797. data/src/core/util/single_set_ptr.h +89 -0
  1798. data/src/core/util/sorted_pack.h +89 -0
  1799. data/src/core/util/spinlock.h +51 -0
  1800. data/src/core/util/stat.h +35 -0
  1801. data/src/core/util/status_helper.cc +431 -0
  1802. data/src/core/util/status_helper.h +160 -0
  1803. data/src/core/util/strerror.cc +40 -0
  1804. data/src/core/util/strerror.h +29 -0
  1805. data/src/core/util/string.cc +341 -0
  1806. data/src/core/util/string.h +110 -0
  1807. data/src/core/util/sync.cc +123 -0
  1808. data/src/core/util/sync.h +199 -0
  1809. data/src/core/util/sync_abseil.cc +107 -0
  1810. data/src/core/util/table.h +486 -0
  1811. data/src/core/util/tchar.cc +49 -0
  1812. data/src/core/util/tchar.h +33 -0
  1813. data/src/core/util/thd.h +193 -0
  1814. data/src/core/util/time.cc +240 -0
  1815. data/src/core/util/time.h +385 -0
  1816. data/src/core/util/time_averaged_stats.cc +60 -0
  1817. data/src/core/util/time_averaged_stats.h +79 -0
  1818. data/src/core/util/time_precise.cc +168 -0
  1819. data/src/core/util/time_precise.h +68 -0
  1820. data/src/core/util/time_util.cc +80 -0
  1821. data/src/core/util/time_util.h +41 -0
  1822. data/src/core/util/tmpfile.h +31 -0
  1823. data/src/core/util/type_list.h +32 -0
  1824. data/src/core/util/unique_ptr_with_bitset.h +86 -0
  1825. data/src/core/util/unique_type_name.h +123 -0
  1826. data/src/core/util/upb_utils.h +43 -0
  1827. data/src/core/util/uri.cc +376 -0
  1828. data/src/core/util/uri.h +105 -0
  1829. data/src/core/util/useful.h +155 -0
  1830. data/src/core/util/uuid_v4.cc +37 -0
  1831. data/src/core/util/uuid_v4.h +35 -0
  1832. data/src/core/util/validation_errors.cc +73 -0
  1833. data/src/core/util/validation_errors.h +144 -0
  1834. data/src/core/util/windows/cpu.cc +34 -0
  1835. data/src/core/util/windows/directory_reader.cc +79 -0
  1836. data/src/core/util/windows/env.cc +56 -0
  1837. data/src/core/util/windows/stat.cc +50 -0
  1838. data/src/core/util/windows/string.cc +68 -0
  1839. data/src/core/util/windows/string_util.cc +53 -0
  1840. data/src/core/util/windows/sync.cc +122 -0
  1841. data/src/core/util/windows/thd.cc +182 -0
  1842. data/src/core/util/windows/time.cc +105 -0
  1843. data/src/core/util/windows/tmpfile.cc +66 -0
  1844. data/src/core/util/work_serializer.cc +538 -0
  1845. data/src/core/util/work_serializer.h +105 -0
  1846. data/src/core/util/xxhash_inline.h +29 -0
  1847. data/src/core/xds/grpc/certificate_provider_store.cc +153 -0
  1848. data/src/core/xds/grpc/certificate_provider_store.h +132 -0
  1849. data/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc +136 -0
  1850. data/src/core/xds/grpc/file_watcher_certificate_provider_factory.h +80 -0
  1851. data/src/core/xds/grpc/xds_audit_logger_registry.cc +123 -0
  1852. data/src/core/xds/grpc/xds_audit_logger_registry.h +67 -0
  1853. data/src/core/xds/grpc/xds_bootstrap_grpc.cc +238 -0
  1854. data/src/core/xds/grpc/xds_bootstrap_grpc.h +167 -0
  1855. data/src/core/xds/grpc/xds_certificate_provider.cc +216 -0
  1856. data/src/core/xds/grpc/xds_certificate_provider.h +116 -0
  1857. data/src/core/xds/grpc/xds_client_grpc.cc +445 -0
  1858. data/src/core/xds/grpc/xds_client_grpc.h +115 -0
  1859. data/src/core/xds/grpc/xds_cluster.cc +79 -0
  1860. data/src/core/xds/grpc/xds_cluster.h +135 -0
  1861. data/src/core/xds/grpc/xds_cluster_parser.cc +772 -0
  1862. data/src/core/xds/grpc/xds_cluster_parser.h +61 -0
  1863. data/src/core/xds/grpc/xds_cluster_specifier_plugin.cc +137 -0
  1864. data/src/core/xds/grpc/xds_cluster_specifier_plugin.h +96 -0
  1865. data/src/core/xds/grpc/xds_common_types.cc +103 -0
  1866. data/src/core/xds/grpc/xds_common_types.h +90 -0
  1867. data/src/core/xds/grpc/xds_common_types_parser.cc +503 -0
  1868. data/src/core/xds/grpc/xds_common_types_parser.h +76 -0
  1869. data/src/core/xds/grpc/xds_endpoint.cc +97 -0
  1870. data/src/core/xds/grpc/xds_endpoint.h +128 -0
  1871. data/src/core/xds/grpc/xds_endpoint_parser.cc +444 -0
  1872. data/src/core/xds/grpc/xds_endpoint_parser.h +47 -0
  1873. data/src/core/xds/grpc/xds_health_status.cc +73 -0
  1874. data/src/core/xds/grpc/xds_health_status.h +88 -0
  1875. data/src/core/xds/grpc/xds_http_fault_filter.cc +249 -0
  1876. data/src/core/xds/grpc/xds_http_fault_filter.h +62 -0
  1877. data/src/core/xds/grpc/xds_http_filter.h +134 -0
  1878. data/src/core/xds/grpc/xds_http_filter_registry.cc +122 -0
  1879. data/src/core/xds/grpc/xds_http_filter_registry.h +104 -0
  1880. data/src/core/xds/grpc/xds_http_gcp_authn_filter.cc +141 -0
  1881. data/src/core/xds/grpc/xds_http_gcp_authn_filter.h +60 -0
  1882. data/src/core/xds/grpc/xds_http_rbac_filter.cc +602 -0
  1883. data/src/core/xds/grpc/xds_http_rbac_filter.h +62 -0
  1884. data/src/core/xds/grpc/xds_http_stateful_session_filter.cc +230 -0
  1885. data/src/core/xds/grpc/xds_http_stateful_session_filter.h +62 -0
  1886. data/src/core/xds/grpc/xds_lb_policy_registry.cc +365 -0
  1887. data/src/core/xds/grpc/xds_lb_policy_registry.h +70 -0
  1888. data/src/core/xds/grpc/xds_listener.cc +175 -0
  1889. data/src/core/xds/grpc/xds_listener.h +203 -0
  1890. data/src/core/xds/grpc/xds_listener_parser.cc +993 -0
  1891. data/src/core/xds/grpc/xds_listener_parser.h +59 -0
  1892. data/src/core/xds/grpc/xds_metadata.cc +62 -0
  1893. data/src/core/xds/grpc/xds_metadata.h +150 -0
  1894. data/src/core/xds/grpc/xds_metadata_parser.cc +184 -0
  1895. data/src/core/xds/grpc/xds_metadata_parser.h +35 -0
  1896. data/src/core/xds/grpc/xds_route_config.cc +282 -0
  1897. data/src/core/xds/grpc/xds_route_config.h +220 -0
  1898. data/src/core/xds/grpc/xds_route_config_parser.cc +962 -0
  1899. data/src/core/xds/grpc/xds_route_config_parser.h +77 -0
  1900. data/src/core/xds/grpc/xds_routing.cc +297 -0
  1901. data/src/core/xds/grpc/xds_routing.h +112 -0
  1902. data/src/core/xds/grpc/xds_server_grpc.cc +160 -0
  1903. data/src/core/xds/grpc/xds_server_grpc.h +63 -0
  1904. data/src/core/xds/grpc/xds_transport_grpc.cc +393 -0
  1905. data/src/core/xds/grpc/xds_transport_grpc.h +149 -0
  1906. data/src/core/xds/xds_client/lrs_client.cc +1292 -0
  1907. data/src/core/xds/xds_client/lrs_client.h +394 -0
  1908. data/src/core/xds/xds_client/xds_api.cc +353 -0
  1909. data/src/core/xds/xds_client/xds_api.h +166 -0
  1910. data/src/core/xds/xds_client/xds_backend_metric_propagation.cc +63 -0
  1911. data/src/core/xds/xds_client/xds_backend_metric_propagation.h +59 -0
  1912. data/src/core/xds/xds_client/xds_bootstrap.cc +37 -0
  1913. data/src/core/xds/xds_client/xds_bootstrap.h +88 -0
  1914. data/src/core/xds/xds_client/xds_channel_args.h +32 -0
  1915. data/src/core/xds/xds_client/xds_client.cc +1689 -0
  1916. data/src/core/xds/xds_client/xds_client.h +334 -0
  1917. data/src/core/xds/xds_client/xds_locality.h +102 -0
  1918. data/src/core/xds/xds_client/xds_metrics.h +41 -0
  1919. data/src/core/xds/xds_client/xds_resource_type.h +97 -0
  1920. data/src/core/xds/xds_client/xds_resource_type_impl.h +87 -0
  1921. data/src/core/xds/xds_client/xds_transport.h +104 -0
  1922. data/src/ruby/bin/math_pb.rb +1 -22
  1923. data/src/ruby/ext/grpc/extconf.rb +1 -4
  1924. data/src/ruby/ext/grpc/rb_byte_buffer.c +3 -3
  1925. data/src/ruby/ext/grpc/rb_call.c +15 -8
  1926. data/src/ruby/ext/grpc/rb_call_credentials.c +39 -32
  1927. data/src/ruby/ext/grpc/rb_call_credentials.h +1 -0
  1928. data/src/ruby/ext/grpc/rb_channel.c +42 -36
  1929. data/src/ruby/ext/grpc/rb_channel_args.c +4 -4
  1930. data/src/ruby/ext/grpc/rb_channel_credentials.c +5 -5
  1931. data/src/ruby/ext/grpc/rb_channel_credentials.h +1 -2
  1932. data/src/ruby/ext/grpc/rb_completion_queue.c +18 -36
  1933. data/src/ruby/ext/grpc/rb_completion_queue.h +7 -1
  1934. data/src/ruby/ext/grpc/rb_compression_options.c +9 -10
  1935. data/src/ruby/ext/grpc/rb_event_thread.c +9 -9
  1936. data/src/ruby/ext/grpc/rb_grpc.c +17 -16
  1937. data/src/ruby/ext/grpc/rb_grpc.h +8 -1
  1938. data/src/ruby/ext/grpc/rb_grpc_imports.generated.c +136 -146
  1939. data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +205 -219
  1940. data/src/ruby/ext/grpc/rb_server.c +47 -27
  1941. data/src/ruby/ext/grpc/rb_server_credentials.c +4 -3
  1942. data/src/ruby/ext/grpc/rb_server_credentials.h +2 -2
  1943. data/src/ruby/ext/grpc/rb_xds_channel_credentials.c +5 -5
  1944. data/src/ruby/ext/grpc/rb_xds_channel_credentials.h +1 -2
  1945. data/src/ruby/ext/grpc/rb_xds_server_credentials.c +5 -4
  1946. data/src/ruby/ext/grpc/rb_xds_server_credentials.h +2 -2
  1947. data/src/ruby/lib/grpc/generic/active_call.rb +16 -7
  1948. data/src/ruby/lib/grpc/logconfig.rb +13 -0
  1949. data/src/ruby/lib/grpc/version.rb +1 -1
  1950. data/src/ruby/pb/grpc/health/v1/health_pb.rb +1 -22
  1951. data/src/ruby/pb/grpc/testing/duplicate/echo_duplicate_services_pb.rb +5 -5
  1952. data/src/ruby/pb/grpc/testing/metrics_pb.rb +10 -19
  1953. data/src/ruby/pb/grpc/testing/metrics_services_pb.rb +5 -5
  1954. data/src/ruby/pb/src/proto/grpc/testing/empty_pb.rb +1 -22
  1955. data/src/ruby/pb/src/proto/grpc/testing/messages_pb.rb +1 -22
  1956. data/src/ruby/pb/src/proto/grpc/testing/test_pb.rb +1 -22
  1957. data/src/ruby/spec/call_spec.rb +53 -40
  1958. data/src/ruby/spec/channel_spec.rb +4 -2
  1959. data/src/ruby/spec/client_server_spec.rb +148 -507
  1960. data/src/ruby/spec/generic/active_call_spec.rb +64 -86
  1961. data/src/ruby/spec/generic/client_stub_spec.rb +20 -20
  1962. data/src/ruby/spec/generic/rpc_server_spec.rb +25 -0
  1963. data/src/ruby/spec/logconfig_spec.rb +30 -0
  1964. data/src/ruby/spec/support/services.rb +3 -0
  1965. data/third_party/abseil-cpp/absl/algorithm/container.h +57 -18
  1966. data/third_party/abseil-cpp/absl/base/attributes.h +84 -0
  1967. data/third_party/abseil-cpp/absl/base/config.h +32 -51
  1968. data/third_party/abseil-cpp/absl/base/dynamic_annotations.h +0 -16
  1969. data/third_party/abseil-cpp/absl/base/internal/nullability_impl.h +3 -1
  1970. data/third_party/abseil-cpp/absl/base/internal/spinlock.h +13 -3
  1971. data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc +0 -12
  1972. data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock_config.h +4 -4
  1973. data/third_party/abseil-cpp/absl/base/macros.h +48 -0
  1974. data/third_party/abseil-cpp/absl/base/no_destructor.h +35 -40
  1975. data/third_party/abseil-cpp/absl/base/nullability.h +33 -7
  1976. data/third_party/abseil-cpp/absl/base/optimization.h +11 -0
  1977. data/third_party/abseil-cpp/absl/base/options.h +1 -1
  1978. data/third_party/abseil-cpp/absl/base/prefetch.h +1 -1
  1979. data/third_party/abseil-cpp/absl/container/flat_hash_map.h +68 -12
  1980. data/third_party/abseil-cpp/absl/container/flat_hash_set.h +60 -6
  1981. data/third_party/abseil-cpp/absl/container/hash_container_defaults.h +45 -0
  1982. data/third_party/abseil-cpp/absl/container/inlined_vector.h +13 -0
  1983. data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h +20 -11
  1984. data/third_party/abseil-cpp/absl/container/internal/compressed_tuple.h +15 -16
  1985. data/third_party/abseil-cpp/absl/container/internal/container_memory.h +34 -1
  1986. data/third_party/abseil-cpp/absl/container/internal/hash_function_defaults.h +68 -1
  1987. data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h +50 -0
  1988. data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc +22 -7
  1989. data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h +26 -8
  1990. data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h +39 -35
  1991. data/third_party/abseil-cpp/absl/container/internal/layout.h +190 -74
  1992. data/third_party/abseil-cpp/absl/container/internal/raw_hash_map.h +8 -6
  1993. data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc +334 -71
  1994. data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h +1299 -458
  1995. data/third_party/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +19 -17
  1996. data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc +4 -3
  1997. data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +2 -1
  1998. data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc +12 -8
  1999. data/third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc +1 -1
  2000. data/third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +11 -7
  2001. data/third_party/abseil-cpp/absl/crc/internal/non_temporal_memcpy.h +18 -17
  2002. data/third_party/abseil-cpp/absl/debugging/internal/bounded_utf8_length_sequence.h +126 -0
  2003. data/third_party/abseil-cpp/absl/debugging/internal/decode_rust_punycode.cc +258 -0
  2004. data/third_party/abseil-cpp/absl/debugging/internal/decode_rust_punycode.h +55 -0
  2005. data/third_party/abseil-cpp/absl/debugging/internal/demangle.cc +1057 -86
  2006. data/third_party/abseil-cpp/absl/debugging/internal/demangle.h +3 -0
  2007. data/third_party/abseil-cpp/absl/debugging/internal/demangle_rust.cc +925 -0
  2008. data/third_party/abseil-cpp/absl/debugging/internal/demangle_rust.h +42 -0
  2009. data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc +43 -16
  2010. data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h +8 -7
  2011. data/third_party/abseil-cpp/absl/debugging/internal/examine_stack.cc +320 -0
  2012. data/third_party/abseil-cpp/absl/debugging/internal/examine_stack.h +64 -0
  2013. data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +10 -7
  2014. data/third_party/abseil-cpp/absl/debugging/internal/utf8_for_code_point.cc +70 -0
  2015. data/third_party/abseil-cpp/absl/debugging/internal/utf8_for_code_point.h +47 -0
  2016. data/third_party/abseil-cpp/absl/flags/commandlineflag.h +11 -0
  2017. data/third_party/abseil-cpp/absl/flags/flag.h +2 -0
  2018. data/third_party/abseil-cpp/absl/flags/internal/flag.cc +117 -30
  2019. data/third_party/abseil-cpp/absl/flags/internal/flag.h +192 -30
  2020. data/third_party/abseil-cpp/absl/flags/reflection.cc +10 -0
  2021. data/third_party/abseil-cpp/absl/functional/any_invocable.h +13 -3
  2022. data/third_party/abseil-cpp/absl/functional/bind_front.h +3 -2
  2023. data/third_party/abseil-cpp/absl/functional/internal/any_invocable.h +7 -7
  2024. data/third_party/abseil-cpp/absl/functional/internal/front_binder.h +9 -9
  2025. data/third_party/abseil-cpp/absl/hash/internal/hash.cc +1 -1
  2026. data/third_party/abseil-cpp/absl/hash/internal/hash.h +13 -3
  2027. data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc +60 -28
  2028. data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.h +4 -0
  2029. data/third_party/abseil-cpp/absl/log/absl_vlog_is_on.h +93 -0
  2030. data/third_party/abseil-cpp/absl/log/check.h +209 -0
  2031. data/third_party/abseil-cpp/absl/log/globals.cc +178 -0
  2032. data/third_party/abseil-cpp/absl/log/globals.h +231 -0
  2033. data/third_party/abseil-cpp/absl/log/internal/append_truncated.h +47 -0
  2034. data/third_party/abseil-cpp/absl/log/internal/check_impl.h +150 -0
  2035. data/third_party/abseil-cpp/absl/log/internal/check_op.cc +138 -0
  2036. data/third_party/abseil-cpp/absl/log/internal/check_op.h +462 -0
  2037. data/third_party/abseil-cpp/absl/log/internal/conditions.cc +83 -0
  2038. data/third_party/abseil-cpp/absl/log/internal/conditions.h +239 -0
  2039. data/third_party/abseil-cpp/absl/log/internal/config.h +45 -0
  2040. data/third_party/abseil-cpp/absl/log/internal/fnmatch.cc +73 -0
  2041. data/third_party/abseil-cpp/absl/log/internal/fnmatch.h +35 -0
  2042. data/third_party/abseil-cpp/absl/log/internal/globals.cc +145 -0
  2043. data/third_party/abseil-cpp/absl/log/internal/globals.h +101 -0
  2044. data/third_party/abseil-cpp/absl/log/internal/log_format.cc +205 -0
  2045. data/third_party/abseil-cpp/absl/log/internal/log_format.h +78 -0
  2046. data/third_party/abseil-cpp/absl/log/internal/log_impl.h +282 -0
  2047. data/third_party/abseil-cpp/absl/log/internal/log_message.cc +690 -0
  2048. data/third_party/abseil-cpp/absl/log/internal/log_message.h +394 -0
  2049. data/third_party/abseil-cpp/absl/log/internal/log_sink_set.cc +296 -0
  2050. data/third_party/abseil-cpp/absl/log/internal/log_sink_set.h +54 -0
  2051. data/third_party/abseil-cpp/absl/log/internal/nullguard.cc +35 -0
  2052. data/third_party/abseil-cpp/absl/log/internal/nullguard.h +88 -0
  2053. data/third_party/abseil-cpp/absl/log/internal/nullstream.h +127 -0
  2054. data/third_party/abseil-cpp/absl/log/internal/proto.cc +220 -0
  2055. data/third_party/abseil-cpp/absl/log/internal/proto.h +288 -0
  2056. data/third_party/abseil-cpp/absl/log/internal/strip.h +108 -0
  2057. data/third_party/abseil-cpp/absl/log/internal/vlog_config.cc +340 -0
  2058. data/third_party/abseil-cpp/absl/log/internal/vlog_config.h +163 -0
  2059. data/third_party/abseil-cpp/absl/log/internal/voidify.h +44 -0
  2060. data/third_party/abseil-cpp/absl/log/log.h +365 -0
  2061. data/third_party/abseil-cpp/absl/log/log_entry.cc +41 -0
  2062. data/third_party/abseil-cpp/absl/log/log_entry.h +221 -0
  2063. data/third_party/abseil-cpp/absl/log/log_sink.cc +23 -0
  2064. data/third_party/abseil-cpp/absl/log/log_sink.h +71 -0
  2065. data/third_party/abseil-cpp/absl/log/log_sink_registry.h +61 -0
  2066. data/third_party/abseil-cpp/absl/log/vlog_is_on.h +72 -0
  2067. data/third_party/abseil-cpp/absl/meta/type_traits.h +138 -42
  2068. data/third_party/abseil-cpp/absl/numeric/int128.cc +0 -3
  2069. data/third_party/abseil-cpp/absl/numeric/int128.h +35 -5
  2070. data/third_party/abseil-cpp/absl/numeric/int128_have_intrinsic.inc +14 -0
  2071. data/third_party/abseil-cpp/absl/numeric/int128_no_intrinsic.inc +18 -0
  2072. data/third_party/abseil-cpp/absl/numeric/internal/bits.h +6 -2
  2073. data/third_party/abseil-cpp/absl/random/beta_distribution.h +8 -8
  2074. data/third_party/abseil-cpp/absl/random/bit_gen_ref.h +9 -7
  2075. data/third_party/abseil-cpp/absl/random/distributions.h +11 -11
  2076. data/third_party/abseil-cpp/absl/random/seed_sequences.h +2 -0
  2077. data/third_party/abseil-cpp/absl/status/internal/statusor_internal.h +61 -2
  2078. data/third_party/abseil-cpp/absl/status/status.cc +0 -4
  2079. data/third_party/abseil-cpp/absl/status/status.h +4 -4
  2080. data/third_party/abseil-cpp/absl/status/statusor.h +108 -142
  2081. data/third_party/abseil-cpp/absl/strings/ascii.cc +32 -71
  2082. data/third_party/abseil-cpp/absl/strings/cord.cc +20 -15
  2083. data/third_party/abseil-cpp/absl/strings/cord.h +68 -7
  2084. data/third_party/abseil-cpp/absl/strings/escaping.cc +96 -21
  2085. data/third_party/abseil-cpp/absl/strings/escaping.h +25 -8
  2086. data/third_party/abseil-cpp/absl/strings/has_absl_stringify.h +1 -0
  2087. data/third_party/abseil-cpp/absl/strings/internal/charconv_bigint.h +10 -0
  2088. data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +29 -9
  2089. data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h +2 -2
  2090. data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.cc +19 -13
  2091. data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.h +23 -13
  2092. data/third_party/abseil-cpp/absl/strings/internal/cordz_handle.cc +24 -24
  2093. data/third_party/abseil-cpp/absl/strings/internal/cordz_info.cc +12 -7
  2094. data/third_party/abseil-cpp/absl/strings/internal/cordz_info.h +9 -4
  2095. data/third_party/abseil-cpp/absl/strings/internal/escaping.cc +5 -2
  2096. data/third_party/abseil-cpp/absl/strings/internal/str_join_internal.h +23 -2
  2097. data/third_party/abseil-cpp/absl/strings/internal/str_split_internal.h +5 -1
  2098. data/third_party/abseil-cpp/absl/strings/numbers.cc +107 -333
  2099. data/third_party/abseil-cpp/absl/strings/numbers.h +12 -151
  2100. data/third_party/abseil-cpp/absl/strings/str_cat.cc +49 -142
  2101. data/third_party/abseil-cpp/absl/strings/str_cat.h +70 -85
  2102. data/third_party/abseil-cpp/absl/strings/str_format.h +1 -1
  2103. data/third_party/abseil-cpp/absl/strings/str_join.h +19 -5
  2104. data/third_party/abseil-cpp/absl/strings/str_split.h +2 -2
  2105. data/third_party/abseil-cpp/absl/strings/string_view.h +3 -2
  2106. data/third_party/abseil-cpp/absl/strings/substitute.cc +4 -0
  2107. data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc +11 -2
  2108. data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.h +5 -0
  2109. data/third_party/abseil-cpp/absl/synchronization/mutex.h +5 -4
  2110. data/third_party/abseil-cpp/absl/time/civil_time.h +2 -2
  2111. data/third_party/abseil-cpp/absl/time/clock.cc +15 -1
  2112. data/third_party/abseil-cpp/absl/time/duration.cc +58 -53
  2113. data/third_party/abseil-cpp/absl/time/format.cc +2 -1
  2114. data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc +1 -1
  2115. data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc +0 -29
  2116. data/third_party/abseil-cpp/absl/time/internal/cctz/src/tzfile.h +3 -3
  2117. data/third_party/abseil-cpp/absl/time/time.h +73 -29
  2118. data/third_party/abseil-cpp/absl/types/compare.h +505 -0
  2119. data/third_party/abseil-cpp/absl/types/internal/optional.h +2 -2
  2120. data/third_party/abseil-cpp/absl/types/internal/variant.h +55 -67
  2121. data/third_party/abseil-cpp/absl/types/optional.h +15 -18
  2122. data/third_party/abseil-cpp/absl/types/span.h +3 -2
  2123. data/third_party/abseil-cpp/absl/types/variant.h +19 -24
  2124. data/third_party/abseil-cpp/absl/utility/utility.h +3 -41
  2125. data/third_party/boringssl-with-bazel/src/crypto/asn1/internal.h +1 -7
  2126. data/third_party/boringssl-with-bazel/src/crypto/asn1/posix_time.c +50 -39
  2127. data/third_party/boringssl-with-bazel/src/crypto/base64/base64.c +4 -0
  2128. data/third_party/boringssl-with-bazel/src/crypto/bcm_support.h +113 -0
  2129. data/third_party/boringssl-with-bazel/src/crypto/bio/bio.c +15 -20
  2130. data/third_party/boringssl-with-bazel/src/crypto/bio/file.c +25 -10
  2131. data/third_party/boringssl-with-bazel/src/crypto/bytestring/cbs.c +3 -18
  2132. data/third_party/boringssl-with-bazel/src/crypto/bytestring/unicode.c +3 -2
  2133. data/third_party/boringssl-with-bazel/src/crypto/chacha/chacha.c +1 -1
  2134. data/third_party/boringssl-with-bazel/src/crypto/chacha/internal.h +19 -15
  2135. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_chacha20poly1305.c +7 -0
  2136. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_des.c +63 -94
  2137. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_null.c +7 -3
  2138. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc2.c +20 -28
  2139. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/e_rc4.c +9 -4
  2140. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/internal.h +43 -0
  2141. data/third_party/boringssl-with-bazel/src/crypto/cipher_extra/tls_cbc.c +2 -2
  2142. data/third_party/boringssl-with-bazel/src/crypto/conf/conf.c +66 -41
  2143. data/third_party/boringssl-with-bazel/src/crypto/cpu_arm_linux.c +4 -1
  2144. data/third_party/boringssl-with-bazel/src/crypto/cpu_intel.c +71 -37
  2145. data/third_party/boringssl-with-bazel/src/crypto/crypto.c +8 -62
  2146. data/third_party/boringssl-with-bazel/src/crypto/curve25519/curve25519.c +7 -7
  2147. data/third_party/boringssl-with-bazel/src/crypto/curve25519/internal.h +1 -1
  2148. data/third_party/boringssl-with-bazel/src/crypto/des/des.c +45 -32
  2149. data/third_party/boringssl-with-bazel/src/crypto/des/internal.h +24 -0
  2150. data/third_party/boringssl-with-bazel/src/crypto/digest_extra/digest_extra.c +1 -0
  2151. data/third_party/boringssl-with-bazel/src/crypto/dilithium/dilithium.c +1539 -0
  2152. data/third_party/boringssl-with-bazel/src/crypto/dilithium/internal.h +58 -0
  2153. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa.c +36 -20
  2154. data/third_party/boringssl-with-bazel/src/crypto/dsa/dsa_asn1.c +3 -4
  2155. data/third_party/boringssl-with-bazel/src/crypto/dsa/internal.h +2 -0
  2156. data/third_party/boringssl-with-bazel/src/crypto/ecdsa_extra/ecdsa_asn1.c +160 -14
  2157. data/third_party/boringssl-with-bazel/src/crypto/err/err.c +81 -60
  2158. data/third_party/boringssl-with-bazel/src/crypto/evp/evp.c +41 -120
  2159. data/third_party/boringssl-with-bazel/src/crypto/evp/evp_asn1.c +13 -13
  2160. data/third_party/boringssl-with-bazel/src/crypto/evp/internal.h +7 -0
  2161. data/third_party/boringssl-with-bazel/src/crypto/evp/p_dh.c +137 -0
  2162. data/third_party/boringssl-with-bazel/src/crypto/evp/p_dh_asn1.c +120 -0
  2163. data/third_party/boringssl-with-bazel/src/crypto/evp/p_dsa_asn1.c +30 -0
  2164. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec.c +3 -4
  2165. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ec_asn1.c +30 -0
  2166. data/third_party/boringssl-with-bazel/src/crypto/evp/p_ed25519.c +1 -4
  2167. data/third_party/boringssl-with-bazel/src/crypto/evp/p_rsa_asn1.c +30 -0
  2168. data/third_party/boringssl-with-bazel/src/crypto/evp/p_x25519.c +1 -4
  2169. data/third_party/boringssl-with-bazel/src/crypto/ex_data.c +6 -7
  2170. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes.c.inc +127 -0
  2171. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/internal.h +33 -7
  2172. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm.c +84 -80
  2173. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bcm_interface.h +89 -0
  2174. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bytes.c.inc +267 -0
  2175. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div.c.inc +876 -0
  2176. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div_extra.c.inc +87 -0
  2177. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c.inc +1320 -0
  2178. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd_extra.c.inc +331 -0
  2179. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/internal.h +33 -23
  2180. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery_inv.c.inc +222 -0
  2181. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/mul.c.inc +744 -0
  2182. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/prime.c.inc +1077 -0
  2183. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/random.c.inc +354 -0
  2184. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/sqrt.c.inc +499 -0
  2185. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/e_aes.c.inc +1464 -0
  2186. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/internal.h +0 -3
  2187. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_key.c.inc +561 -0
  2188. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p256-nistz.c.inc +734 -0
  2189. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p256-nistz.h +65 -8
  2190. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/scalar.c.inc +173 -0
  2191. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/ecdsa.c.inc +317 -0
  2192. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/internal.h +28 -11
  2193. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/fips_shared_support.c +3 -6
  2194. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/internal.h +1 -80
  2195. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/rand.c.inc +479 -0
  2196. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/padding.c.inc +420 -0
  2197. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa.c.inc +1038 -0
  2198. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa_impl.c.inc +1351 -0
  2199. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/fips.c.inc +121 -0
  2200. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/self_check.c.inc +1038 -0
  2201. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/internal.h +2 -2
  2202. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/service_indicator.c.inc +328 -0
  2203. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/internal.h +78 -29
  2204. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1.c.inc +439 -0
  2205. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha256.c.inc +359 -0
  2206. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c.inc +543 -0
  2207. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/tls/kdf.c.inc +218 -0
  2208. data/third_party/boringssl-with-bazel/src/crypto/hpke/hpke.c +295 -2
  2209. data/third_party/boringssl-with-bazel/src/crypto/internal.h +143 -45
  2210. data/third_party/boringssl-with-bazel/src/crypto/kyber/internal.h +1 -1
  2211. data/third_party/boringssl-with-bazel/src/crypto/kyber/kyber.c +2 -1
  2212. data/third_party/boringssl-with-bazel/src/crypto/mem.c +25 -12
  2213. data/third_party/boringssl-with-bazel/src/crypto/mldsa/internal.h +73 -0
  2214. data/third_party/boringssl-with-bazel/src/crypto/mldsa/mldsa.c +1687 -0
  2215. data/third_party/boringssl-with-bazel/src/crypto/mlkem/internal.h +90 -0
  2216. data/third_party/boringssl-with-bazel/src/crypto/mlkem/mlkem.cc +1097 -0
  2217. data/third_party/boringssl-with-bazel/src/crypto/obj/obj_dat.h +4 -1
  2218. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_lib.c +19 -19
  2219. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pk8.c +43 -41
  2220. data/third_party/boringssl-with-bazel/src/crypto/pem/pem_pkey.c +13 -12
  2221. data/third_party/boringssl-with-bazel/src/crypto/pkcs8/internal.h +1 -0
  2222. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/deterministic.c +9 -1
  2223. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/fork_detect.c +195 -0
  2224. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/getentropy.c +9 -1
  2225. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/ios.c +9 -1
  2226. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/passive.c +19 -3
  2227. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/rand_extra.c +26 -23
  2228. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/sysrand_internal.h +37 -0
  2229. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/trusty.c +9 -1
  2230. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/urandom.c +328 -0
  2231. data/third_party/boringssl-with-bazel/src/crypto/rand_extra/windows.c +8 -1
  2232. data/third_party/boringssl-with-bazel/src/crypto/rsa_extra/internal.h +2 -0
  2233. data/third_party/boringssl-with-bazel/src/crypto/spx/spx.c +7 -6
  2234. data/third_party/boringssl-with-bazel/src/crypto/x509/by_dir.c +26 -33
  2235. data/third_party/boringssl-with-bazel/src/crypto/x509/by_file.c +1 -1
  2236. data/third_party/boringssl-with-bazel/src/crypto/x509/internal.h +8 -5
  2237. data/third_party/boringssl-with-bazel/src/crypto/x509/rsa_pss.c +5 -1
  2238. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_cpols.c +9 -4
  2239. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_crld.c +2 -2
  2240. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_info.c +1 -1
  2241. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_lib.c +2 -0
  2242. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_purp.c +40 -61
  2243. data/third_party/boringssl-with-bazel/src/crypto/x509/v3_utl.c +49 -16
  2244. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_lu.c +24 -17
  2245. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_trs.c +51 -78
  2246. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vfy.c +12 -29
  2247. data/third_party/boringssl-with-bazel/src/crypto/x509/x509_vpm.c +13 -2
  2248. data/third_party/boringssl-with-bazel/src/crypto/x509/x_algor.c +2 -2
  2249. data/third_party/boringssl-with-bazel/src/crypto/x509/x_x509.c +1 -6
  2250. data/third_party/boringssl-with-bazel/src/gen/crypto/err_data.c +1514 -0
  2251. data/third_party/boringssl-with-bazel/src/include/openssl/asn1.h +24 -11
  2252. data/third_party/boringssl-with-bazel/src/include/openssl/base.h +10 -2
  2253. data/third_party/boringssl-with-bazel/src/include/openssl/bio.h +88 -45
  2254. data/third_party/boringssl-with-bazel/src/include/openssl/bn.h +14 -9
  2255. data/third_party/boringssl-with-bazel/src/include/openssl/bytestring.h +12 -4
  2256. data/third_party/boringssl-with-bazel/src/include/openssl/crypto.h +10 -11
  2257. data/third_party/boringssl-with-bazel/src/include/openssl/dh.h +17 -1
  2258. data/third_party/boringssl-with-bazel/src/include/openssl/dsa.h +10 -3
  2259. data/third_party/boringssl-with-bazel/src/include/openssl/err.h +13 -0
  2260. data/third_party/boringssl-with-bazel/src/include/openssl/evp.h +25 -14
  2261. data/third_party/boringssl-with-bazel/src/include/openssl/evp_errors.h +1 -0
  2262. data/third_party/boringssl-with-bazel/src/include/openssl/ex_data.h +15 -3
  2263. data/third_party/boringssl-with-bazel/src/include/openssl/experimental/dilithium.h +129 -0
  2264. data/third_party/boringssl-with-bazel/src/include/openssl/experimental/kyber.h +146 -0
  2265. data/third_party/boringssl-with-bazel/src/include/openssl/experimental/spx.h +90 -0
  2266. data/third_party/boringssl-with-bazel/src/include/openssl/hpke.h +8 -6
  2267. data/third_party/boringssl-with-bazel/src/include/openssl/mldsa.h +136 -0
  2268. data/third_party/boringssl-with-bazel/src/include/openssl/mlkem.h +246 -0
  2269. data/third_party/boringssl-with-bazel/src/include/openssl/nid.h +3 -0
  2270. data/third_party/boringssl-with-bazel/src/include/openssl/obj.h +2 -2
  2271. data/third_party/boringssl-with-bazel/src/include/openssl/pem.h +66 -57
  2272. data/third_party/boringssl-with-bazel/src/include/openssl/posix_time.h +6 -0
  2273. data/third_party/boringssl-with-bazel/src/include/openssl/rand.h +19 -10
  2274. data/third_party/boringssl-with-bazel/src/include/openssl/rsa.h +19 -10
  2275. data/third_party/boringssl-with-bazel/src/include/openssl/service_indicator.h +2 -2
  2276. data/third_party/boringssl-with-bazel/src/include/openssl/span.h +62 -12
  2277. data/third_party/boringssl-with-bazel/src/include/openssl/ssl.h +409 -111
  2278. data/third_party/boringssl-with-bazel/src/include/openssl/stack.h +10 -3
  2279. data/third_party/boringssl-with-bazel/src/include/openssl/target.h +13 -10
  2280. data/third_party/boringssl-with-bazel/src/include/openssl/tls1.h +2 -2
  2281. data/third_party/boringssl-with-bazel/src/include/openssl/x509.h +2927 -2394
  2282. data/third_party/boringssl-with-bazel/src/ssl/d1_both.cc +9 -12
  2283. data/third_party/boringssl-with-bazel/src/ssl/d1_lib.cc +6 -0
  2284. data/third_party/boringssl-with-bazel/src/ssl/d1_pkt.cc +10 -5
  2285. data/third_party/boringssl-with-bazel/src/ssl/dtls_method.cc +14 -2
  2286. data/third_party/boringssl-with-bazel/src/ssl/dtls_record.cc +301 -66
  2287. data/third_party/boringssl-with-bazel/src/ssl/encrypted_client_hello.cc +3 -9
  2288. data/third_party/boringssl-with-bazel/src/ssl/extensions.cc +66 -58
  2289. data/third_party/boringssl-with-bazel/src/ssl/handoff.cc +1 -1
  2290. data/third_party/boringssl-with-bazel/src/ssl/handshake.cc +17 -7
  2291. data/third_party/boringssl-with-bazel/src/ssl/handshake_client.cc +138 -54
  2292. data/third_party/boringssl-with-bazel/src/ssl/handshake_server.cc +159 -117
  2293. data/third_party/boringssl-with-bazel/src/ssl/internal.h +349 -202
  2294. data/third_party/boringssl-with-bazel/src/ssl/s3_both.cc +45 -17
  2295. data/third_party/boringssl-with-bazel/src/ssl/s3_lib.cc +0 -1
  2296. data/third_party/boringssl-with-bazel/src/ssl/s3_pkt.cc +21 -1
  2297. data/third_party/boringssl-with-bazel/src/ssl/ssl_aead_ctx.cc +86 -1
  2298. data/third_party/boringssl-with-bazel/src/ssl/ssl_buffer.cc +7 -4
  2299. data/third_party/boringssl-with-bazel/src/ssl/ssl_cert.cc +66 -385
  2300. data/third_party/boringssl-with-bazel/src/ssl/ssl_cipher.cc +21 -19
  2301. data/third_party/boringssl-with-bazel/src/ssl/ssl_credential.cc +423 -0
  2302. data/third_party/boringssl-with-bazel/src/ssl/ssl_file.cc +2 -2
  2303. data/third_party/boringssl-with-bazel/src/ssl/ssl_key_share.cc +99 -4
  2304. data/third_party/boringssl-with-bazel/src/ssl/ssl_lib.cc +139 -83
  2305. data/third_party/boringssl-with-bazel/src/ssl/ssl_privkey.cc +116 -93
  2306. data/third_party/boringssl-with-bazel/src/ssl/ssl_session.cc +6 -14
  2307. data/third_party/boringssl-with-bazel/src/ssl/ssl_versions.cc +7 -1
  2308. data/third_party/boringssl-with-bazel/src/ssl/ssl_x509.cc +48 -116
  2309. data/third_party/boringssl-with-bazel/src/ssl/tls13_both.cc +21 -27
  2310. data/third_party/boringssl-with-bazel/src/ssl/tls13_client.cc +54 -7
  2311. data/third_party/boringssl-with-bazel/src/ssl/tls13_enc.cc +96 -34
  2312. data/third_party/boringssl-with-bazel/src/ssl/tls13_server.cc +68 -23
  2313. data/third_party/boringssl-with-bazel/src/ssl/tls_record.cc +4 -24
  2314. data/third_party/upb/upb/base/descriptor_constants.h +29 -2
  2315. data/third_party/upb/upb/base/internal/endian.h +46 -0
  2316. data/third_party/upb/upb/base/status.h +1 -1
  2317. data/third_party/upb/upb/base/status.hpp +5 -1
  2318. data/third_party/upb/upb/base/string_view.h +7 -5
  2319. data/third_party/upb/upb/base/upcast.h +29 -0
  2320. data/third_party/upb/upb/generated_code_support.h +2 -1
  2321. data/third_party/upb/upb/json/decode.c +161 -64
  2322. data/third_party/upb/upb/json/decode.h +29 -29
  2323. data/third_party/upb/upb/json/encode.c +5 -28
  2324. data/third_party/upb/upb/json/encode.h +3 -26
  2325. data/third_party/upb/upb/lex/round_trip.c +10 -0
  2326. data/third_party/upb/upb/mem/arena.c +312 -109
  2327. data/third_party/upb/upb/mem/arena.h +27 -68
  2328. data/third_party/upb/upb/mem/arena.hpp +13 -5
  2329. data/third_party/upb/upb/mem/internal/arena.h +78 -57
  2330. data/third_party/upb/upb/message/accessors.c +17 -45
  2331. data/third_party/upb/upb/message/accessors.h +209 -314
  2332. data/third_party/upb/upb/message/array.c +78 -83
  2333. data/third_party/upb/upb/message/array.h +26 -30
  2334. data/third_party/upb/upb/message/compat.c +39 -0
  2335. data/third_party/upb/upb/message/compat.h +41 -0
  2336. data/third_party/upb/upb/message/copy.c +325 -0
  2337. data/third_party/upb/upb/message/copy.h +56 -0
  2338. data/third_party/upb/upb/message/internal/accessors.h +738 -176
  2339. data/third_party/upb/upb/message/internal/array.h +86 -79
  2340. data/third_party/upb/upb/message/internal/compare_unknown.c +289 -0
  2341. data/third_party/upb/upb/message/internal/compare_unknown.h +49 -0
  2342. data/third_party/upb/upb/message/internal/extension.c +63 -0
  2343. data/third_party/upb/upb/message/internal/extension.h +14 -17
  2344. data/third_party/upb/upb/message/internal/map.h +43 -41
  2345. data/third_party/upb/upb/message/internal/map_entry.h +14 -39
  2346. data/third_party/upb/upb/message/internal/map_sorter.h +19 -39
  2347. data/third_party/upb/upb/message/internal/message.c +75 -0
  2348. data/third_party/upb/upb/message/internal/message.h +36 -41
  2349. data/third_party/upb/upb/message/internal/tagged_ptr.h +56 -0
  2350. data/third_party/upb/upb/message/internal/types.h +46 -11
  2351. data/third_party/upb/upb/message/map.c +28 -26
  2352. data/third_party/upb/upb/message/map.h +16 -40
  2353. data/third_party/upb/upb/message/map_gencode_util.h +3 -26
  2354. data/third_party/upb/upb/message/map_sorter.c +22 -34
  2355. data/third_party/upb/upb/message/merge.c +38 -0
  2356. data/third_party/upb/upb/message/merge.h +26 -0
  2357. data/third_party/upb/upb/message/message.c +106 -103
  2358. data/third_party/upb/upb/message/message.h +19 -8
  2359. data/third_party/upb/upb/message/tagged_ptr.h +15 -36
  2360. data/third_party/upb/upb/message/value.h +29 -5
  2361. data/third_party/upb/upb/mini_descriptor/build_enum.c +23 -14
  2362. data/third_party/upb/upb/mini_descriptor/build_enum.h +6 -14
  2363. data/third_party/upb/upb/mini_descriptor/decode.c +145 -109
  2364. data/third_party/upb/upb/mini_descriptor/decode.h +3 -8
  2365. data/third_party/upb/upb/mini_descriptor/link.c +47 -33
  2366. data/third_party/upb/upb/mini_descriptor/link.h +1 -1
  2367. data/third_party/upb/upb/mini_table/enum.h +4 -8
  2368. data/third_party/upb/upb/mini_table/extension.h +29 -0
  2369. data/third_party/upb/upb/mini_table/extension_registry.c +27 -2
  2370. data/third_party/upb/upb/mini_table/extension_registry.h +17 -0
  2371. data/third_party/upb/upb/mini_table/field.h +28 -68
  2372. data/third_party/upb/upb/mini_table/file.h +31 -0
  2373. data/third_party/upb/upb/mini_table/internal/enum.h +19 -20
  2374. data/third_party/upb/upb/mini_table/internal/extension.h +43 -3
  2375. data/third_party/upb/upb/mini_table/internal/field.h +137 -28
  2376. data/third_party/upb/upb/mini_table/internal/file.h +47 -10
  2377. data/third_party/upb/upb/mini_table/internal/message.c +37 -9
  2378. data/third_party/upb/upb/mini_table/internal/message.h +138 -23
  2379. data/third_party/upb/upb/mini_table/internal/size_log2.h +77 -0
  2380. data/third_party/upb/upb/mini_table/internal/sub.h +44 -4
  2381. data/third_party/upb/upb/mini_table/message.c +18 -19
  2382. data/third_party/upb/upb/mini_table/message.h +35 -25
  2383. data/third_party/upb/upb/mini_table/sub.h +32 -1
  2384. data/third_party/upb/upb/port/def.inc +125 -11
  2385. data/third_party/upb/upb/port/undef.inc +11 -1
  2386. data/third_party/upb/upb/reflection/common.h +1 -11
  2387. data/third_party/upb/upb/reflection/def.hpp +42 -3
  2388. data/third_party/upb/upb/reflection/def_pool.c +81 -2
  2389. data/third_party/upb/upb/reflection/def_pool.h +10 -2
  2390. data/third_party/upb/upb/reflection/descriptor_bootstrap.h +19 -0
  2391. data/third_party/upb/upb/reflection/enum_def.c +46 -19
  2392. data/third_party/upb/upb/reflection/enum_def.h +2 -0
  2393. data/third_party/upb/upb/reflection/enum_value_def.c +38 -14
  2394. data/third_party/upb/upb/reflection/enum_value_def.h +2 -0
  2395. data/third_party/upb/upb/reflection/extension_range.c +13 -5
  2396. data/third_party/upb/upb/reflection/extension_range.h +2 -0
  2397. data/third_party/upb/upb/reflection/field_def.c +214 -141
  2398. data/third_party/upb/upb/reflection/field_def.h +14 -1
  2399. data/third_party/upb/upb/reflection/file_def.c +124 -9
  2400. data/third_party/upb/upb/reflection/file_def.h +6 -0
  2401. data/third_party/upb/upb/reflection/internal/def_builder.c +85 -7
  2402. data/third_party/upb/upb/reflection/internal/def_builder.h +23 -0
  2403. data/third_party/upb/upb/reflection/internal/enum_def.h +5 -4
  2404. data/third_party/upb/upb/reflection/internal/enum_reserved_range.h +1 -1
  2405. data/third_party/upb/upb/reflection/internal/enum_value_def.h +2 -1
  2406. data/third_party/upb/upb/reflection/internal/extension_range.h +2 -2
  2407. data/third_party/upb/upb/reflection/internal/field_def.h +11 -10
  2408. data/third_party/upb/upb/reflection/internal/message_def.h +6 -3
  2409. data/third_party/upb/upb/reflection/internal/method_def.h +5 -3
  2410. data/third_party/upb/upb/reflection/internal/oneof_def.h +5 -3
  2411. data/third_party/upb/upb/reflection/internal/service_def.h +5 -3
  2412. data/third_party/upb/upb/reflection/internal/upb_edition_defaults.h +20 -0
  2413. data/third_party/upb/upb/reflection/message.c +52 -20
  2414. data/third_party/upb/upb/reflection/message.h +7 -7
  2415. data/third_party/upb/upb/reflection/message_def.c +68 -34
  2416. data/third_party/upb/upb/reflection/message_def.h +2 -0
  2417. data/third_party/upb/upb/reflection/method_def.c +20 -9
  2418. data/third_party/upb/upb/reflection/method_def.h +10 -7
  2419. data/third_party/upb/upb/reflection/oneof_def.c +20 -11
  2420. data/third_party/upb/upb/reflection/oneof_def.h +3 -1
  2421. data/third_party/upb/upb/reflection/service_def.c +25 -13
  2422. data/third_party/upb/upb/reflection/service_def.h +8 -5
  2423. data/third_party/upb/upb/text/encode.c +65 -311
  2424. data/third_party/upb/upb/text/encode.h +1 -11
  2425. data/third_party/upb/upb/text/internal/encode.c +180 -0
  2426. data/third_party/upb/upb/text/internal/encode.h +240 -0
  2427. data/third_party/upb/upb/text/options.h +22 -0
  2428. data/third_party/upb/upb/wire/decode.c +353 -269
  2429. data/third_party/upb/upb/wire/decode.h +25 -2
  2430. data/third_party/upb/upb/wire/encode.c +160 -105
  2431. data/third_party/upb/upb/wire/encode.h +16 -4
  2432. data/third_party/upb/upb/wire/eps_copy_input_stream.h +3 -3
  2433. data/third_party/upb/upb/wire/internal/decode_fast.c +1000 -0
  2434. data/third_party/upb/upb/wire/internal/decode_fast.h +148 -0
  2435. data/third_party/upb/upb/wire/internal/decoder.h +127 -0
  2436. data/third_party/upb/upb/wire/internal/reader.h +61 -0
  2437. data/third_party/upb/upb/wire/reader.c +10 -8
  2438. data/third_party/upb/upb/wire/reader.h +20 -53
  2439. data/third_party/utf8_range/utf8_range.c +467 -0
  2440. data/third_party/utf8_range/utf8_range.h +9 -8
  2441. metadata +541 -399
  2442. data/src/core/client_channel/backend_metric.cc +0 -95
  2443. data/src/core/client_channel/backend_metric.h +0 -47
  2444. data/src/core/client_channel/channel_connectivity.cc +0 -265
  2445. data/src/core/client_channel/client_channel_channelz.cc +0 -93
  2446. data/src/core/client_channel/client_channel_channelz.h +0 -85
  2447. data/src/core/client_channel/config_selector.cc +0 -60
  2448. data/src/core/client_channel/http_proxy_mapper.cc +0 -305
  2449. data/src/core/client_channel/http_proxy_mapper.h +0 -53
  2450. data/src/core/client_channel/service_config_channel_arg_filter.cc +0 -136
  2451. data/src/core/ext/filters/channel_idle/channel_idle_filter.cc +0 -318
  2452. data/src/core/ext/filters/channel_idle/channel_idle_filter.h +0 -150
  2453. data/src/core/ext/filters/deadline/deadline_filter.cc +0 -407
  2454. data/src/core/ext/filters/deadline/deadline_filter.h +0 -85
  2455. data/src/core/ext/filters/http/message_compress/legacy_compression_filter.cc +0 -325
  2456. data/src/core/ext/filters/http/message_compress/legacy_compression_filter.h +0 -139
  2457. data/src/core/ext/filters/server_config_selector/server_config_selector.h +0 -83
  2458. data/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +0 -170
  2459. data/src/core/ext/filters/server_config_selector/server_config_selector_filter.h +0 -33
  2460. data/src/core/ext/gcp/metadata_query.cc +0 -136
  2461. data/src/core/ext/gcp/metadata_query.h +0 -87
  2462. data/src/core/ext/transport/chttp2/transport/http_trace.cc +0 -19
  2463. data/src/core/ext/transport/chttp2/transport/http_trace.h +0 -24
  2464. data/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.cc +0 -44
  2465. data/src/core/ext/transport/chttp2/transport/max_concurrent_streams_policy.h +0 -67
  2466. data/src/core/ext/transport/inproc/inproc_plugin.cc +0 -23
  2467. data/src/core/ext/xds/certificate_provider_store.cc +0 -155
  2468. data/src/core/ext/xds/certificate_provider_store.h +0 -133
  2469. data/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +0 -138
  2470. data/src/core/ext/xds/file_watcher_certificate_provider_factory.h +0 -82
  2471. data/src/core/ext/xds/upb_utils.h +0 -45
  2472. data/src/core/ext/xds/xds_api.cc +0 -572
  2473. data/src/core/ext/xds/xds_api.h +0 -189
  2474. data/src/core/ext/xds/xds_audit_logger_registry.cc +0 -122
  2475. data/src/core/ext/xds/xds_audit_logger_registry.h +0 -68
  2476. data/src/core/ext/xds/xds_bootstrap.cc +0 -38
  2477. data/src/core/ext/xds/xds_bootstrap.h +0 -90
  2478. data/src/core/ext/xds/xds_bootstrap_grpc.cc +0 -365
  2479. data/src/core/ext/xds/xds_bootstrap_grpc.h +0 -182
  2480. data/src/core/ext/xds/xds_certificate_provider.cc +0 -217
  2481. data/src/core/ext/xds/xds_certificate_provider.h +0 -116
  2482. data/src/core/ext/xds/xds_channel_args.h +0 -32
  2483. data/src/core/ext/xds/xds_channel_stack_modifier.cc +0 -107
  2484. data/src/core/ext/xds/xds_channel_stack_modifier.h +0 -63
  2485. data/src/core/ext/xds/xds_client.cc +0 -2123
  2486. data/src/core/ext/xds/xds_client.h +0 -358
  2487. data/src/core/ext/xds/xds_client_grpc.cc +0 -281
  2488. data/src/core/ext/xds/xds_client_grpc.h +0 -96
  2489. data/src/core/ext/xds/xds_client_stats.cc +0 -173
  2490. data/src/core/ext/xds/xds_client_stats.h +0 -251
  2491. data/src/core/ext/xds/xds_cluster.cc +0 -795
  2492. data/src/core/ext/xds/xds_cluster.h +0 -142
  2493. data/src/core/ext/xds/xds_cluster_specifier_plugin.cc +0 -140
  2494. data/src/core/ext/xds/xds_cluster_specifier_plugin.h +0 -97
  2495. data/src/core/ext/xds/xds_common_types.cc +0 -509
  2496. data/src/core/ext/xds/xds_common_types.h +0 -108
  2497. data/src/core/ext/xds/xds_endpoint.cc +0 -516
  2498. data/src/core/ext/xds/xds_endpoint.h +0 -150
  2499. data/src/core/ext/xds/xds_health_status.cc +0 -75
  2500. data/src/core/ext/xds/xds_health_status.h +0 -91
  2501. data/src/core/ext/xds/xds_http_fault_filter.cc +0 -238
  2502. data/src/core/ext/xds/xds_http_fault_filter.h +0 -58
  2503. data/src/core/ext/xds/xds_http_filters.cc +0 -117
  2504. data/src/core/ext/xds/xds_http_filters.h +0 -182
  2505. data/src/core/ext/xds/xds_http_rbac_filter.cc +0 -588
  2506. data/src/core/ext/xds/xds_http_rbac_filter.h +0 -58
  2507. data/src/core/ext/xds/xds_http_stateful_session_filter.cc +0 -218
  2508. data/src/core/ext/xds/xds_http_stateful_session_filter.h +0 -58
  2509. data/src/core/ext/xds/xds_lb_policy_registry.cc +0 -371
  2510. data/src/core/ext/xds/xds_lb_policy_registry.h +0 -71
  2511. data/src/core/ext/xds/xds_listener.cc +0 -1142
  2512. data/src/core/ext/xds/xds_listener.h +0 -236
  2513. data/src/core/ext/xds/xds_resource_type.h +0 -98
  2514. data/src/core/ext/xds/xds_resource_type_impl.h +0 -88
  2515. data/src/core/ext/xds/xds_route_config.cc +0 -1198
  2516. data/src/core/ext/xds/xds_route_config.h +0 -253
  2517. data/src/core/ext/xds/xds_routing.cc +0 -264
  2518. data/src/core/ext/xds/xds_routing.h +0 -106
  2519. data/src/core/ext/xds/xds_server_config_fetcher.cc +0 -1392
  2520. data/src/core/ext/xds/xds_transport.h +0 -89
  2521. data/src/core/ext/xds/xds_transport_grpc.cc +0 -365
  2522. data/src/core/ext/xds/xds_transport_grpc.h +0 -139
  2523. data/src/core/lib/avl/avl.h +0 -325
  2524. data/src/core/lib/backoff/backoff.cc +0 -47
  2525. data/src/core/lib/backoff/backoff.h +0 -89
  2526. data/src/core/lib/backoff/random_early_detection.cc +0 -33
  2527. data/src/core/lib/backoff/random_early_detection.h +0 -62
  2528. data/src/core/lib/channel/call_tracer.cc +0 -365
  2529. data/src/core/lib/channel/call_tracer.h +0 -225
  2530. data/src/core/lib/channel/channel_stack_trace.cc +0 -19
  2531. data/src/core/lib/channel/channel_stack_trace.h +0 -24
  2532. data/src/core/lib/channel/channel_trace.cc +0 -187
  2533. data/src/core/lib/channel/channel_trace.h +0 -138
  2534. data/src/core/lib/channel/channelz.cc +0 -610
  2535. data/src/core/lib/channel/channelz.h +0 -381
  2536. data/src/core/lib/channel/channelz_registry.cc +0 -281
  2537. data/src/core/lib/channel/channelz_registry.h +0 -102
  2538. data/src/core/lib/channel/context.h +0 -77
  2539. data/src/core/lib/channel/server_call_tracer_filter.cc +0 -111
  2540. data/src/core/lib/channel/tcp_tracer.h +0 -140
  2541. data/src/core/lib/config/config_vars.cc +0 -152
  2542. data/src/core/lib/config/config_vars.h +0 -127
  2543. data/src/core/lib/config/config_vars_non_generated.cc +0 -49
  2544. data/src/core/lib/config/core_configuration.cc +0 -111
  2545. data/src/core/lib/config/core_configuration.h +0 -244
  2546. data/src/core/lib/config/load_config.cc +0 -79
  2547. data/src/core/lib/config/load_config.h +0 -55
  2548. data/src/core/lib/debug/event_log.cc +0 -88
  2549. data/src/core/lib/debug/event_log.h +0 -81
  2550. data/src/core/lib/debug/histogram_view.cc +0 -69
  2551. data/src/core/lib/debug/histogram_view.h +0 -37
  2552. data/src/core/lib/debug/stats.cc +0 -68
  2553. data/src/core/lib/debug/stats.h +0 -64
  2554. data/src/core/lib/debug/stats_data.cc +0 -509
  2555. data/src/core/lib/debug/stats_data.h +0 -403
  2556. data/src/core/lib/event_engine/trace.cc +0 -25
  2557. data/src/core/lib/event_engine/trace.h +0 -49
  2558. data/src/core/lib/gpr/alloc.cc +0 -71
  2559. data/src/core/lib/gpr/alloc.h +0 -28
  2560. data/src/core/lib/gpr/android/log.cc +0 -79
  2561. data/src/core/lib/gpr/atm.cc +0 -35
  2562. data/src/core/lib/gpr/iphone/cpu.cc +0 -44
  2563. data/src/core/lib/gpr/linux/cpu.cc +0 -87
  2564. data/src/core/lib/gpr/linux/log.cc +0 -114
  2565. data/src/core/lib/gpr/log.cc +0 -140
  2566. data/src/core/lib/gpr/log_internal.h +0 -55
  2567. data/src/core/lib/gpr/msys/tmpfile.cc +0 -59
  2568. data/src/core/lib/gpr/posix/cpu.cc +0 -84
  2569. data/src/core/lib/gpr/posix/log.cc +0 -111
  2570. data/src/core/lib/gpr/posix/string.cc +0 -72
  2571. data/src/core/lib/gpr/posix/sync.cc +0 -159
  2572. data/src/core/lib/gpr/posix/time.cc +0 -182
  2573. data/src/core/lib/gpr/posix/tmpfile.cc +0 -71
  2574. data/src/core/lib/gpr/spinlock.h +0 -53
  2575. data/src/core/lib/gpr/string.cc +0 -344
  2576. data/src/core/lib/gpr/string.h +0 -112
  2577. data/src/core/lib/gpr/sync.cc +0 -124
  2578. data/src/core/lib/gpr/sync_abseil.cc +0 -110
  2579. data/src/core/lib/gpr/time.cc +0 -269
  2580. data/src/core/lib/gpr/time_precise.cc +0 -168
  2581. data/src/core/lib/gpr/time_precise.h +0 -69
  2582. data/src/core/lib/gpr/tmpfile.h +0 -32
  2583. data/src/core/lib/gpr/useful.h +0 -184
  2584. data/src/core/lib/gpr/windows/cpu.cc +0 -35
  2585. data/src/core/lib/gpr/windows/log.cc +0 -116
  2586. data/src/core/lib/gpr/windows/string.cc +0 -69
  2587. data/src/core/lib/gpr/windows/string_util.cc +0 -55
  2588. data/src/core/lib/gpr/windows/sync.cc +0 -122
  2589. data/src/core/lib/gpr/windows/time.cc +0 -105
  2590. data/src/core/lib/gpr/windows/tmpfile.cc +0 -68
  2591. data/src/core/lib/gpr/wrap_memcpy.cc +0 -43
  2592. data/src/core/lib/gprpp/atomic_utils.h +0 -47
  2593. data/src/core/lib/gprpp/bitset.h +0 -225
  2594. data/src/core/lib/gprpp/chunked_vector.h +0 -257
  2595. data/src/core/lib/gprpp/construct_destruct.h +0 -40
  2596. data/src/core/lib/gprpp/cpp_impl_of.h +0 -49
  2597. data/src/core/lib/gprpp/crash.cc +0 -43
  2598. data/src/core/lib/gprpp/crash.h +0 -37
  2599. data/src/core/lib/gprpp/debug_location.h +0 -99
  2600. data/src/core/lib/gprpp/directory_reader.h +0 -48
  2601. data/src/core/lib/gprpp/down_cast.h +0 -49
  2602. data/src/core/lib/gprpp/dual_ref_counted.h +0 -356
  2603. data/src/core/lib/gprpp/env.h +0 -53
  2604. data/src/core/lib/gprpp/examine_stack.cc +0 -43
  2605. data/src/core/lib/gprpp/examine_stack.h +0 -45
  2606. data/src/core/lib/gprpp/fork.cc +0 -242
  2607. data/src/core/lib/gprpp/fork.h +0 -98
  2608. data/src/core/lib/gprpp/host_port.cc +0 -114
  2609. data/src/core/lib/gprpp/host_port.h +0 -57
  2610. data/src/core/lib/gprpp/if_list.h +0 -4530
  2611. data/src/core/lib/gprpp/linux/env.cc +0 -62
  2612. data/src/core/lib/gprpp/load_file.cc +0 -76
  2613. data/src/core/lib/gprpp/load_file.h +0 -34
  2614. data/src/core/lib/gprpp/manual_constructor.h +0 -146
  2615. data/src/core/lib/gprpp/match.h +0 -75
  2616. data/src/core/lib/gprpp/memory.h +0 -53
  2617. data/src/core/lib/gprpp/mpscq.cc +0 -108
  2618. data/src/core/lib/gprpp/mpscq.h +0 -99
  2619. data/src/core/lib/gprpp/no_destruct.h +0 -95
  2620. data/src/core/lib/gprpp/notification.h +0 -67
  2621. data/src/core/lib/gprpp/orphanable.h +0 -150
  2622. data/src/core/lib/gprpp/overload.h +0 -59
  2623. data/src/core/lib/gprpp/packed_table.h +0 -40
  2624. data/src/core/lib/gprpp/per_cpu.cc +0 -35
  2625. data/src/core/lib/gprpp/per_cpu.h +0 -104
  2626. data/src/core/lib/gprpp/posix/directory_reader.cc +0 -82
  2627. data/src/core/lib/gprpp/posix/env.cc +0 -47
  2628. data/src/core/lib/gprpp/posix/stat.cc +0 -54
  2629. data/src/core/lib/gprpp/posix/thd.cc +0 -247
  2630. data/src/core/lib/gprpp/ref_counted.h +0 -391
  2631. data/src/core/lib/gprpp/ref_counted_ptr.h +0 -444
  2632. data/src/core/lib/gprpp/ref_counted_string.cc +0 -44
  2633. data/src/core/lib/gprpp/ref_counted_string.h +0 -161
  2634. data/src/core/lib/gprpp/single_set_ptr.h +0 -87
  2635. data/src/core/lib/gprpp/sorted_pack.h +0 -89
  2636. data/src/core/lib/gprpp/stat.h +0 -36
  2637. data/src/core/lib/gprpp/status_helper.cc +0 -453
  2638. data/src/core/lib/gprpp/status_helper.h +0 -190
  2639. data/src/core/lib/gprpp/strerror.cc +0 -41
  2640. data/src/core/lib/gprpp/strerror.h +0 -29
  2641. data/src/core/lib/gprpp/sync.h +0 -200
  2642. data/src/core/lib/gprpp/table.h +0 -452
  2643. data/src/core/lib/gprpp/tchar.cc +0 -49
  2644. data/src/core/lib/gprpp/tchar.h +0 -33
  2645. data/src/core/lib/gprpp/thd.h +0 -195
  2646. data/src/core/lib/gprpp/time.cc +0 -242
  2647. data/src/core/lib/gprpp/time.h +0 -379
  2648. data/src/core/lib/gprpp/time_averaged_stats.cc +0 -60
  2649. data/src/core/lib/gprpp/time_averaged_stats.h +0 -79
  2650. data/src/core/lib/gprpp/time_util.cc +0 -81
  2651. data/src/core/lib/gprpp/time_util.h +0 -42
  2652. data/src/core/lib/gprpp/type_list.h +0 -32
  2653. data/src/core/lib/gprpp/unique_type_name.h +0 -104
  2654. data/src/core/lib/gprpp/validation_errors.cc +0 -65
  2655. data/src/core/lib/gprpp/validation_errors.h +0 -134
  2656. data/src/core/lib/gprpp/windows/directory_reader.cc +0 -81
  2657. data/src/core/lib/gprpp/windows/env.cc +0 -56
  2658. data/src/core/lib/gprpp/windows/stat.cc +0 -50
  2659. data/src/core/lib/gprpp/windows/thd.cc +0 -182
  2660. data/src/core/lib/gprpp/work_serializer.cc +0 -558
  2661. data/src/core/lib/gprpp/work_serializer.h +0 -107
  2662. data/src/core/lib/gprpp/xxhash_inline.h +0 -29
  2663. data/src/core/lib/handshaker/proxy_mapper.h +0 -53
  2664. data/src/core/lib/handshaker/proxy_mapper_registry.cc +0 -71
  2665. data/src/core/lib/handshaker/proxy_mapper_registry.h +0 -75
  2666. data/src/core/lib/http/format_request.cc +0 -137
  2667. data/src/core/lib/http/format_request.h +0 -38
  2668. data/src/core/lib/http/httpcli.cc +0 -392
  2669. data/src/core/lib/http/httpcli.h +0 -268
  2670. data/src/core/lib/http/httpcli_security_connector.cc +0 -213
  2671. data/src/core/lib/http/httpcli_ssl_credentials.h +0 -39
  2672. data/src/core/lib/http/parser.cc +0 -451
  2673. data/src/core/lib/http/parser.h +0 -130
  2674. data/src/core/lib/iomgr/ev_windows.cc +0 -30
  2675. data/src/core/lib/iomgr/gethostname.h +0 -26
  2676. data/src/core/lib/iomgr/gethostname_fallback.cc +0 -30
  2677. data/src/core/lib/iomgr/gethostname_host_name_max.cc +0 -40
  2678. data/src/core/lib/iomgr/gethostname_sysconf.cc +0 -40
  2679. data/src/core/lib/iomgr/grpc_if_nametoindex.h +0 -30
  2680. data/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc +0 -43
  2681. data/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc +0 -39
  2682. data/src/core/lib/json/json.h +0 -30
  2683. data/src/core/lib/json/json_args.h +0 -34
  2684. data/src/core/lib/json/json_channel_args.h +0 -42
  2685. data/src/core/lib/json/json_object_loader.cc +0 -216
  2686. data/src/core/lib/json/json_object_loader.h +0 -645
  2687. data/src/core/lib/json/json_reader.cc +0 -956
  2688. data/src/core/lib/json/json_reader.h +0 -34
  2689. data/src/core/lib/json/json_util.cc +0 -101
  2690. data/src/core/lib/json/json_util.h +0 -164
  2691. data/src/core/lib/json/json_writer.cc +0 -339
  2692. data/src/core/lib/json/json_writer.h +0 -33
  2693. data/src/core/lib/matchers/matchers.cc +0 -330
  2694. data/src/core/lib/matchers/matchers.h +0 -163
  2695. data/src/core/lib/promise/trace.cc +0 -20
  2696. data/src/core/lib/promise/trace.h +0 -24
  2697. data/src/core/lib/resource_quota/trace.cc +0 -19
  2698. data/src/core/lib/resource_quota/trace.h +0 -24
  2699. data/src/core/lib/security/transport/legacy_server_auth_filter.cc +0 -244
  2700. data/src/core/lib/security/transport/secure_endpoint.cc +0 -565
  2701. data/src/core/lib/security/transport/secure_endpoint.h +0 -43
  2702. data/src/core/lib/security/transport/security_handshaker.cc +0 -673
  2703. data/src/core/lib/security/transport/security_handshaker.h +0 -45
  2704. data/src/core/lib/security/transport/tsi_error.cc +0 -31
  2705. data/src/core/lib/security/transport/tsi_error.h +0 -30
  2706. data/src/core/lib/slice/b64.cc +0 -239
  2707. data/src/core/lib/slice/b64.h +0 -52
  2708. data/src/core/lib/slice/slice_refcount.cc +0 -20
  2709. data/src/core/lib/surface/api_trace.cc +0 -25
  2710. data/src/core/lib/surface/api_trace.h +0 -53
  2711. data/src/core/lib/surface/builtins.cc +0 -37
  2712. data/src/core/lib/surface/builtins.h +0 -26
  2713. data/src/core/lib/surface/call_trace.cc +0 -163
  2714. data/src/core/lib/surface/call_trace.h +0 -30
  2715. data/src/core/lib/surface/channel_ping.cc +0 -69
  2716. data/src/core/lib/surface/server.cc +0 -2045
  2717. data/src/core/lib/surface/server.h +0 -551
  2718. data/src/core/lib/surface/wait_for_cq_end_op.cc +0 -75
  2719. data/src/core/lib/surface/wait_for_cq_end_op.h +0 -72
  2720. data/src/core/lib/transport/batch_builder.cc +0 -179
  2721. data/src/core/lib/transport/batch_builder.h +0 -478
  2722. data/src/core/lib/transport/call_factory.cc +0 -41
  2723. data/src/core/lib/transport/call_factory.h +0 -56
  2724. data/src/core/lib/transport/call_size_estimator.cc +0 -41
  2725. data/src/core/lib/transport/call_size_estimator.h +0 -52
  2726. data/src/core/lib/transport/handshaker.cc +0 -229
  2727. data/src/core/lib/transport/handshaker.h +0 -165
  2728. data/src/core/lib/transport/handshaker_factory.h +0 -74
  2729. data/src/core/lib/transport/handshaker_registry.cc +0 -61
  2730. data/src/core/lib/transport/handshaker_registry.h +0 -69
  2731. data/src/core/lib/transport/http_connect_handshaker.cc +0 -400
  2732. data/src/core/lib/transport/http_connect_handshaker.h +0 -42
  2733. data/src/core/lib/transport/tcp_connect_handshaker.cc +0 -244
  2734. data/src/core/lib/transport/tcp_connect_handshaker.h +0 -39
  2735. data/src/core/lib/uri/uri_parser.cc +0 -372
  2736. data/src/core/lib/uri/uri_parser.h +0 -101
  2737. data/src/core/load_balancing/subchannel_list.h +0 -455
  2738. data/src/core/resolver/binder/binder_resolver.cc +0 -147
  2739. data/src/core/resolver/xds/xds_resolver_trace.cc +0 -25
  2740. data/src/core/resolver/xds/xds_resolver_trace.h +0 -30
  2741. data/third_party/abseil-cpp/absl/strings/internal/has_absl_stringify.h +0 -44
  2742. data/third_party/boringssl-with-bazel/err_data.c +0 -1512
  2743. data/third_party/boringssl-with-bazel/src/crypto/conf/conf_def.h +0 -122
  2744. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/aes.c +0 -106
  2745. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/bytes.c +0 -267
  2746. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div.c +0 -909
  2747. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/div_extra.c +0 -87
  2748. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/exponentiation.c +0 -1276
  2749. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/gcd_extra.c +0 -328
  2750. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/montgomery_inv.c +0 -222
  2751. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/mul.c +0 -744
  2752. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/prime.c +0 -1070
  2753. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/random.c +0 -355
  2754. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/sqrt.c +0 -499
  2755. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/e_aes.c +0 -1463
  2756. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/ec_key.c +0 -553
  2757. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/p256-nistz.c +0 -637
  2758. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/scalar.c +0 -169
  2759. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdsa/ecdsa.c +0 -372
  2760. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.c +0 -197
  2761. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/fork_detect.h +0 -67
  2762. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/rand.c +0 -493
  2763. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/urandom.c +0 -328
  2764. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/padding.c +0 -423
  2765. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa.c +0 -1038
  2766. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/rsa_impl.c +0 -1337
  2767. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/fips.c +0 -119
  2768. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/self_check/self_check.c +0 -1064
  2769. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/service_indicator/service_indicator.c +0 -331
  2770. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha1.c +0 -439
  2771. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha256.c +0 -359
  2772. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/sha/sha512.c +0 -537
  2773. data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/tls/kdf.c +0 -211
  2774. data/third_party/boringssl-with-bazel/src/crypto/spx/internal.h +0 -79
  2775. data/third_party/boringssl-with-bazel/src/include/openssl/kyber.h +0 -136
  2776. data/third_party/upb/upb/message/types.h +0 -15
  2777. data/third_party/upb/upb/wire/decode_fast.c +0 -996
  2778. data/third_party/upb/upb/wire/decode_fast.h +0 -147
  2779. data/third_party/upb/upb/wire/internal/decode.h +0 -143
  2780. data/third_party/upb/upb/wire/internal/swap.h +0 -45
  2781. data/third_party/utf8_range/naive.c +0 -92
  2782. data/third_party/utf8_range/range2-neon.c +0 -157
  2783. data/third_party/utf8_range/range2-sse.c +0 -170
  2784. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/{aes_nohw.c → aes_nohw.c.inc} +0 -0
  2785. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/{key_wrap.c → key_wrap.c.inc} +0 -0
  2786. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/aes/{mode_wrappers.c → mode_wrappers.c.inc} +0 -0
  2787. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{add.c → add.c.inc} +0 -0
  2788. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/asm/{x86_64-gcc.c → x86_64-gcc.c.inc} +0 -0
  2789. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{bn.c → bn.c.inc} +0 -0
  2790. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{cmp.c → cmp.c.inc} +0 -0
  2791. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{ctx.c → ctx.c.inc} +0 -0
  2792. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{gcd.c → gcd.c.inc} +0 -0
  2793. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{generic.c → generic.c.inc} +0 -0
  2794. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{jacobi.c → jacobi.c.inc} +0 -0
  2795. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{montgomery.c → montgomery.c.inc} +0 -0
  2796. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{rsaz_exp.c → rsaz_exp.c.inc} +0 -0
  2797. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/bn/{shift.c → shift.c.inc} +0 -0
  2798. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/{aead.c → aead.c.inc} +0 -0
  2799. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/{cipher.c → cipher.c.inc} +0 -0
  2800. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cipher/{e_aesccm.c → e_aesccm.c.inc} +0 -0
  2801. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/cmac/{cmac.c → cmac.c.inc} +0 -0
  2802. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/{check.c → check.c.inc} +0 -0
  2803. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/dh/{dh.c → dh.c.inc} +0 -0
  2804. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/digest/{digest.c → digest.c.inc} +0 -0
  2805. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/digest/{digests.c → digests.c.inc} +0 -0
  2806. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/digestsign/{digestsign.c → digestsign.c.inc} +0 -0
  2807. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{ec.c → ec.c.inc} +0 -0
  2808. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{ec_montgomery.c → ec_montgomery.c.inc} +0 -0
  2809. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{felem.c → felem.c.inc} +0 -0
  2810. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{oct.c → oct.c.inc} +0 -0
  2811. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p224-64.c → p224-64.c.inc} +0 -0
  2812. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{p256.c → p256.c.inc} +0 -0
  2813. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{simple.c → simple.c.inc} +0 -0
  2814. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{simple_mul.c → simple_mul.c.inc} +0 -0
  2815. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{util.c → util.c.inc} +0 -0
  2816. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ec/{wnaf.c → wnaf.c.inc} +0 -0
  2817. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/ecdh/{ecdh.c → ecdh.c.inc} +0 -0
  2818. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/hkdf/{hkdf.c → hkdf.c.inc} +0 -0
  2819. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/hmac/{hmac.c → hmac.c.inc} +0 -0
  2820. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/md4/{md4.c → md4.c.inc} +0 -0
  2821. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/md5/{md5.c → md5.c.inc} +0 -0
  2822. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{cbc.c → cbc.c.inc} +0 -0
  2823. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{cfb.c → cfb.c.inc} +0 -0
  2824. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{ctr.c → ctr.c.inc} +0 -0
  2825. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{gcm.c → gcm.c.inc} +0 -0
  2826. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{gcm_nohw.c → gcm_nohw.c.inc} +0 -0
  2827. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{ofb.c → ofb.c.inc} +0 -0
  2828. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/modes/{polyval.c → polyval.c.inc} +0 -0
  2829. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rand/{ctrdrbg.c → ctrdrbg.c.inc} +0 -0
  2830. /data/third_party/boringssl-with-bazel/src/crypto/fipsmodule/rsa/{blinding.c → blinding.c.inc} +0 -0
  2831. /data/third_party/boringssl-with-bazel/src/crypto/{fipsmodule/rand → rand_extra}/getrandom_fillin.h +0 -0
@@ -80,7 +80,7 @@
80
80
  // slot_type slots[capacity];
81
81
  // };
82
82
  //
83
- // The length of this array is computed by `AllocSize()` below.
83
+ // The length of this array is computed by `RawHashSetLayout::alloc_size` below.
84
84
  //
85
85
  // Control bytes (`ctrl_t`) are bytes (collected into groups of a
86
86
  // platform-specific size) that define the state of the corresponding slot in
@@ -100,6 +100,13 @@
100
100
  // Storing control bytes in a separate array also has beneficial cache effects,
101
101
  // since more logical slots will fit into a cache line.
102
102
  //
103
+ // # Small Object Optimization (SOO)
104
+ //
105
+ // When the size/alignment of the value_type and the capacity of the table are
106
+ // small, we enable small object optimization and store the values inline in
107
+ // the raw_hash_set object. This optimization allows us to avoid
108
+ // allocation/deallocation as well as cache/dTLB misses.
109
+ //
103
110
  // # Hashing
104
111
  //
105
112
  // We compute two separate hashes, `H1` and `H2`, from the hash of an object.
@@ -233,9 +240,10 @@ namespace container_internal {
233
240
 
234
241
  #ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS
235
242
  #error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set
236
- #elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \
237
- defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \
238
- defined(ABSL_HAVE_MEMORY_SANITIZER)
243
+ #elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \
244
+ defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \
245
+ defined(ABSL_HAVE_MEMORY_SANITIZER)) && \
246
+ !defined(NDEBUG_SANITIZER) // If defined, performance is important.
239
247
  // When compiled in sanitizer mode, we add generation integers to the backing
240
248
  // array and iterators. In the backing array, we store the generation between
241
249
  // the control bytes and the slots. When iterators are dereferenced, we assert
@@ -374,6 +382,9 @@ uint32_t TrailingZeros(T x) {
374
382
  return static_cast<uint32_t>(countr_zero(x));
375
383
  }
376
384
 
385
+ // 8 bytes bitmask with most significant bit set for every byte.
386
+ constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL;
387
+
377
388
  // An abstract bitmask, such as that emitted by a SIMD instruction.
378
389
  //
379
390
  // Specifically, this type implements a simple bitset whose representation is
@@ -423,27 +434,35 @@ class NonIterableBitMask {
423
434
  // an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When
424
435
  // `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as
425
436
  // the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask.
437
+ // If NullifyBitsOnIteration is true (only allowed for Shift == 3),
438
+ // non zero abstract bit is allowed to have additional bits
439
+ // (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not).
426
440
  //
427
441
  // For example:
428
442
  // for (int i : BitMask<uint32_t, 16>(0b101)) -> yields 0, 2
429
443
  // for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3
430
- template <class T, int SignificantBits, int Shift = 0>
444
+ template <class T, int SignificantBits, int Shift = 0,
445
+ bool NullifyBitsOnIteration = false>
431
446
  class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> {
432
447
  using Base = NonIterableBitMask<T, SignificantBits, Shift>;
433
448
  static_assert(std::is_unsigned<T>::value, "");
434
449
  static_assert(Shift == 0 || Shift == 3, "");
450
+ static_assert(!NullifyBitsOnIteration || Shift == 3, "");
435
451
 
436
452
  public:
437
- explicit BitMask(T mask) : Base(mask) {}
453
+ explicit BitMask(T mask) : Base(mask) {
454
+ if (Shift == 3 && !NullifyBitsOnIteration) {
455
+ assert(this->mask_ == (this->mask_ & kMsbs8Bytes));
456
+ }
457
+ }
438
458
  // BitMask is an iterator over the indices of its abstract bits.
439
459
  using value_type = int;
440
460
  using iterator = BitMask;
441
461
  using const_iterator = BitMask;
442
462
 
443
463
  BitMask& operator++() {
444
- if (Shift == 3) {
445
- constexpr uint64_t msbs = 0x8080808080808080ULL;
446
- this->mask_ &= msbs;
464
+ if (Shift == 3 && NullifyBitsOnIteration) {
465
+ this->mask_ &= kMsbs8Bytes;
447
466
  }
448
467
  this->mask_ &= (this->mask_ - 1);
449
468
  return *this;
@@ -520,10 +539,24 @@ ABSL_DLL extern const ctrl_t kEmptyGroup[32];
520
539
  // Returns a pointer to a control byte group that can be used by empty tables.
521
540
  inline ctrl_t* EmptyGroup() {
522
541
  // Const must be cast away here; no uses of this function will actually write
523
- // to it, because it is only used for empty tables.
542
+ // to it because it is only used for empty tables.
524
543
  return const_cast<ctrl_t*>(kEmptyGroup + 16);
525
544
  }
526
545
 
546
+ // For use in SOO iterators.
547
+ // TODO(b/289225379): we could potentially get rid of this by adding an is_soo
548
+ // bit in iterators. This would add branches but reduce cache misses.
549
+ ABSL_DLL extern const ctrl_t kSooControl[17];
550
+
551
+ // Returns a pointer to a full byte followed by a sentinel byte.
552
+ inline ctrl_t* SooControl() {
553
+ // Const must be cast away here; no uses of this function will actually write
554
+ // to it because it is only used for SOO iterators.
555
+ return const_cast<ctrl_t*>(kSooControl);
556
+ }
557
+ // Whether ctrl is from the SooControl array.
558
+ inline bool IsSooControl(const ctrl_t* ctrl) { return ctrl == SooControl(); }
559
+
527
560
  // Returns a pointer to a generation to use for an empty hashtable.
528
561
  GenerationType* EmptyGeneration();
529
562
 
@@ -535,7 +568,37 @@ inline bool IsEmptyGeneration(const GenerationType* generation) {
535
568
 
536
569
  // Mixes a randomly generated per-process seed with `hash` and `ctrl` to
537
570
  // randomize insertion order within groups.
538
- bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl);
571
+ bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash,
572
+ const ctrl_t* ctrl);
573
+
574
+ ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ShouldInsertBackwards(
575
+ ABSL_ATTRIBUTE_UNUSED size_t capacity, ABSL_ATTRIBUTE_UNUSED size_t hash,
576
+ ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) {
577
+ #if defined(NDEBUG)
578
+ return false;
579
+ #else
580
+ return ShouldInsertBackwardsForDebug(capacity, hash, ctrl);
581
+ #endif
582
+ }
583
+
584
+ // Returns insert position for the given mask.
585
+ // We want to add entropy even when ASLR is not enabled.
586
+ // In debug build we will randomly insert in either the front or back of
587
+ // the group.
588
+ // TODO(kfm,sbenza): revisit after we do unconditional mixing
589
+ template <class Mask>
590
+ ABSL_ATTRIBUTE_ALWAYS_INLINE inline auto GetInsertionOffset(
591
+ Mask mask, ABSL_ATTRIBUTE_UNUSED size_t capacity,
592
+ ABSL_ATTRIBUTE_UNUSED size_t hash,
593
+ ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) {
594
+ #if defined(NDEBUG)
595
+ return mask.LowestBitSet();
596
+ #else
597
+ return ShouldInsertBackwardsForDebug(capacity, hash, ctrl)
598
+ ? mask.HighestBitSet()
599
+ : mask.LowestBitSet();
600
+ #endif
601
+ }
539
602
 
540
603
  // Returns a per-table, hash salt, which changes on resize. This gets mixed into
541
604
  // H1 to randomize iteration order per-table.
@@ -560,7 +623,12 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; }
560
623
 
561
624
  // Helpers for checking the state of a control byte.
562
625
  inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; }
563
- inline bool IsFull(ctrl_t c) { return c >= static_cast<ctrl_t>(0); }
626
+ inline bool IsFull(ctrl_t c) {
627
+ // Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0`
628
+ // is not a value in the enum. Both ways are equivalent, but this way makes
629
+ // linters happier.
630
+ return static_cast<std::underlying_type_t<ctrl_t>>(c) >= 0;
631
+ }
564
632
  inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; }
565
633
  inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; }
566
634
 
@@ -646,6 +714,14 @@ struct GroupSse2Impl {
646
714
  static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff));
647
715
  }
648
716
 
717
+ // Returns a bitmask representing the positions of non full slots.
718
+ // Note: this includes: kEmpty, kDeleted, kSentinel.
719
+ // It is useful in contexts when kSentinel is not present.
720
+ auto MaskNonFull() const {
721
+ return BitMask<uint16_t, kWidth>(
722
+ static_cast<uint16_t>(_mm_movemask_epi8(ctrl)));
723
+ }
724
+
649
725
  // Returns a bitmask representing the positions of empty or deleted slots.
650
726
  NonIterableBitMask<uint16_t, kWidth> MaskEmptyOrDeleted() const {
651
727
  auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
@@ -685,10 +761,11 @@ struct GroupAArch64Impl {
685
761
  ctrl = vld1_u8(reinterpret_cast<const uint8_t*>(pos));
686
762
  }
687
763
 
688
- BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const {
764
+ auto Match(h2_t hash) const {
689
765
  uint8x8_t dup = vdup_n_u8(hash);
690
766
  auto mask = vceq_u8(ctrl, dup);
691
- return BitMask<uint64_t, kWidth, 3>(
767
+ return BitMask<uint64_t, kWidth, /*Shift=*/3,
768
+ /*NullifyBitsOnIteration=*/true>(
692
769
  vget_lane_u64(vreinterpret_u64_u8(mask), 0));
693
770
  }
694
771
 
@@ -704,12 +781,25 @@ struct GroupAArch64Impl {
704
781
  // Returns a bitmask representing the positions of full slots.
705
782
  // Note: for `is_small()` tables group may contain the "same" slot twice:
706
783
  // original and mirrored.
707
- BitMask<uint64_t, kWidth, 3> MaskFull() const {
784
+ auto MaskFull() const {
708
785
  uint64_t mask = vget_lane_u64(
709
786
  vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl),
710
787
  vdup_n_s8(static_cast<int8_t>(0)))),
711
788
  0);
712
- return BitMask<uint64_t, kWidth, 3>(mask);
789
+ return BitMask<uint64_t, kWidth, /*Shift=*/3,
790
+ /*NullifyBitsOnIteration=*/true>(mask);
791
+ }
792
+
793
+ // Returns a bitmask representing the positions of non full slots.
794
+ // Note: this includes: kEmpty, kDeleted, kSentinel.
795
+ // It is useful in contexts when kSentinel is not present.
796
+ auto MaskNonFull() const {
797
+ uint64_t mask = vget_lane_u64(
798
+ vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl),
799
+ vdup_n_s8(static_cast<int8_t>(0)))),
800
+ 0);
801
+ return BitMask<uint64_t, kWidth, /*Shift=*/3,
802
+ /*NullifyBitsOnIteration=*/true>(mask);
713
803
  }
714
804
 
715
805
  NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
@@ -736,11 +826,10 @@ struct GroupAArch64Impl {
736
826
 
737
827
  void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
738
828
  uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0);
739
- constexpr uint64_t msbs = 0x8080808080808080ULL;
740
829
  constexpr uint64_t slsbs = 0x0202020202020202ULL;
741
830
  constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL;
742
831
  auto x = slsbs & (mask >> 6);
743
- auto res = (x + midbs) | msbs;
832
+ auto res = (x + midbs) | kMsbs8Bytes;
744
833
  little_endian::Store64(dst, res);
745
834
  }
746
835
 
@@ -768,30 +857,33 @@ struct GroupPortableImpl {
768
857
  // v = 0x1716151413121110
769
858
  // hash = 0x12
770
859
  // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000
771
- constexpr uint64_t msbs = 0x8080808080808080ULL;
772
860
  constexpr uint64_t lsbs = 0x0101010101010101ULL;
773
861
  auto x = ctrl ^ (lsbs * hash);
774
- return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs);
862
+ return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & kMsbs8Bytes);
775
863
  }
776
864
 
777
865
  NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const {
778
- constexpr uint64_t msbs = 0x8080808080808080ULL;
779
866
  return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 6)) &
780
- msbs);
867
+ kMsbs8Bytes);
781
868
  }
782
869
 
783
870
  // Returns a bitmask representing the positions of full slots.
784
871
  // Note: for `is_small()` tables group may contain the "same" slot twice:
785
872
  // original and mirrored.
786
873
  BitMask<uint64_t, kWidth, 3> MaskFull() const {
787
- constexpr uint64_t msbs = 0x8080808080808080ULL;
788
- return BitMask<uint64_t, kWidth, 3>((ctrl ^ msbs) & msbs);
874
+ return BitMask<uint64_t, kWidth, 3>((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes);
875
+ }
876
+
877
+ // Returns a bitmask representing the positions of non full slots.
878
+ // Note: this includes: kEmpty, kDeleted, kSentinel.
879
+ // It is useful in contexts when kSentinel is not present.
880
+ auto MaskNonFull() const {
881
+ return BitMask<uint64_t, kWidth, 3>(ctrl & kMsbs8Bytes);
789
882
  }
790
883
 
791
884
  NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
792
- constexpr uint64_t msbs = 0x8080808080808080ULL;
793
885
  return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 7)) &
794
- msbs);
886
+ kMsbs8Bytes);
795
887
  }
796
888
 
797
889
  uint32_t CountLeadingEmptyOrDeleted() const {
@@ -803,9 +895,8 @@ struct GroupPortableImpl {
803
895
  }
804
896
 
805
897
  void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
806
- constexpr uint64_t msbs = 0x8080808080808080ULL;
807
898
  constexpr uint64_t lsbs = 0x0101010101010101ULL;
808
- auto x = ctrl & msbs;
899
+ auto x = ctrl & kMsbs8Bytes;
809
900
  auto res = (~x + (x >> 7)) & ~lsbs;
810
901
  little_endian::Store64(dst, res);
811
902
  }
@@ -815,21 +906,21 @@ struct GroupPortableImpl {
815
906
 
816
907
  #ifdef ABSL_INTERNAL_HAVE_SSE2
817
908
  using Group = GroupSse2Impl;
818
- using GroupEmptyOrDeleted = GroupSse2Impl;
909
+ using GroupFullEmptyOrDeleted = GroupSse2Impl;
819
910
  #elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN)
820
911
  using Group = GroupAArch64Impl;
821
912
  // For Aarch64, we use the portable implementation for counting and masking
822
- // empty or deleted group elements. This is to avoid the latency of moving
913
+ // full, empty or deleted group elements. This is to avoid the latency of moving
823
914
  // between data GPRs and Neon registers when it does not provide a benefit.
824
915
  // Using Neon is profitable when we call Match(), but is not when we don't,
825
- // which is the case when we do *EmptyOrDeleted operations. It is difficult to
826
- // make a similar approach beneficial on other architectures such as x86 since
827
- // they have much lower GPR <-> vector register transfer latency and 16-wide
828
- // Groups.
829
- using GroupEmptyOrDeleted = GroupPortableImpl;
916
+ // which is the case when we do *EmptyOrDeleted and MaskFull operations.
917
+ // It is difficult to make a similar approach beneficial on other architectures
918
+ // such as x86 since they have much lower GPR <-> vector register transfer
919
+ // latency and 16-wide Groups.
920
+ using GroupFullEmptyOrDeleted = GroupPortableImpl;
830
921
  #else
831
922
  using Group = GroupPortableImpl;
832
- using GroupEmptyOrDeleted = GroupPortableImpl;
923
+ using GroupFullEmptyOrDeleted = GroupPortableImpl;
833
924
  #endif
834
925
 
835
926
  // When there is an insertion with no reserved growth, we rehash with
@@ -978,17 +1069,96 @@ using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled;
978
1069
  using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled;
979
1070
  #endif
980
1071
 
1072
+ // Stored the information regarding number of slots we can still fill
1073
+ // without needing to rehash.
1074
+ //
1075
+ // We want to ensure sufficient number of empty slots in the table in order
1076
+ // to keep probe sequences relatively short. Empty slot in the probe group
1077
+ // is required to stop probing.
1078
+ //
1079
+ // Tombstones (kDeleted slots) are not included in the growth capacity,
1080
+ // because we'd like to rehash when the table is filled with tombstones and/or
1081
+ // full slots.
1082
+ //
1083
+ // GrowthInfo also stores a bit that encodes whether table may have any
1084
+ // deleted slots.
1085
+ // Most of the tables (>95%) have no deleted slots, so some functions can
1086
+ // be more efficient with this information.
1087
+ //
1088
+ // Callers can also force a rehash via the standard `rehash(0)`,
1089
+ // which will recompute this value as a side-effect.
1090
+ //
1091
+ // See also `CapacityToGrowth()`.
1092
+ class GrowthInfo {
1093
+ public:
1094
+ // Leaves data member uninitialized.
1095
+ GrowthInfo() = default;
1096
+
1097
+ // Initializes the GrowthInfo assuming we can grow `growth_left` elements
1098
+ // and there are no kDeleted slots in the table.
1099
+ void InitGrowthLeftNoDeleted(size_t growth_left) {
1100
+ growth_left_info_ = growth_left;
1101
+ }
1102
+
1103
+ // Overwrites single full slot with an empty slot.
1104
+ void OverwriteFullAsEmpty() { ++growth_left_info_; }
1105
+
1106
+ // Overwrites single empty slot with a full slot.
1107
+ void OverwriteEmptyAsFull() {
1108
+ assert(GetGrowthLeft() > 0);
1109
+ --growth_left_info_;
1110
+ }
1111
+
1112
+ // Overwrites several empty slots with full slots.
1113
+ void OverwriteManyEmptyAsFull(size_t cnt) {
1114
+ assert(GetGrowthLeft() >= cnt);
1115
+ growth_left_info_ -= cnt;
1116
+ }
1117
+
1118
+ // Overwrites specified control element with full slot.
1119
+ void OverwriteControlAsFull(ctrl_t ctrl) {
1120
+ assert(GetGrowthLeft() >= static_cast<size_t>(IsEmpty(ctrl)));
1121
+ growth_left_info_ -= static_cast<size_t>(IsEmpty(ctrl));
1122
+ }
1123
+
1124
+ // Overwrites single full slot with a deleted slot.
1125
+ void OverwriteFullAsDeleted() { growth_left_info_ |= kDeletedBit; }
1126
+
1127
+ // Returns true if table satisfies two properties:
1128
+ // 1. Guaranteed to have no kDeleted slots.
1129
+ // 2. There is a place for at least one element to grow.
1130
+ bool HasNoDeletedAndGrowthLeft() const {
1131
+ return static_cast<std::make_signed_t<size_t>>(growth_left_info_) > 0;
1132
+ }
1133
+
1134
+ // Returns true if the table satisfies two properties:
1135
+ // 1. Guaranteed to have no kDeleted slots.
1136
+ // 2. There is no growth left.
1137
+ bool HasNoGrowthLeftAndNoDeleted() const { return growth_left_info_ == 0; }
1138
+
1139
+ // Returns true if table guaranteed to have no k
1140
+ bool HasNoDeleted() const {
1141
+ return static_cast<std::make_signed_t<size_t>>(growth_left_info_) >= 0;
1142
+ }
1143
+
1144
+ // Returns the number of elements left to grow.
1145
+ size_t GetGrowthLeft() const { return growth_left_info_ & kGrowthLeftMask; }
1146
+
1147
+ private:
1148
+ static constexpr size_t kGrowthLeftMask = ((~size_t{}) >> 1);
1149
+ static constexpr size_t kDeletedBit = ~kGrowthLeftMask;
1150
+ // Topmost bit signal whenever there are deleted slots.
1151
+ size_t growth_left_info_;
1152
+ };
1153
+
1154
+ static_assert(sizeof(GrowthInfo) == sizeof(size_t), "");
1155
+ static_assert(alignof(GrowthInfo) == alignof(size_t), "");
1156
+
981
1157
  // Returns whether `n` is a valid capacity (i.e., number of slots).
982
1158
  //
983
1159
  // A valid capacity is a non-zero integer `2^m - 1`.
984
1160
  inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; }
985
1161
 
986
- // Computes the offset from the start of the backing allocation of control.
987
- // infoz and growth_left are stored at the beginning of the backing array.
988
- inline size_t ControlOffset(bool has_infoz) {
989
- return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t);
990
- }
991
-
992
1162
  // Returns the number of "cloned control bytes".
993
1163
  //
994
1164
  // This is the number of control bytes that are present both at the beginning
@@ -996,36 +1166,157 @@ inline size_t ControlOffset(bool has_infoz) {
996
1166
  // `Group::kWidth`-width probe window starting from any control byte.
997
1167
  constexpr size_t NumClonedBytes() { return Group::kWidth - 1; }
998
1168
 
999
- // Given the capacity of a table, computes the offset (from the start of the
1000
- // backing allocation) of the generation counter (if it exists).
1001
- inline size_t GenerationOffset(size_t capacity, bool has_infoz) {
1002
- assert(IsValidCapacity(capacity));
1003
- const size_t num_control_bytes = capacity + 1 + NumClonedBytes();
1004
- return ControlOffset(has_infoz) + num_control_bytes;
1169
+ // Returns the number of control bytes including cloned.
1170
+ constexpr size_t NumControlBytes(size_t capacity) {
1171
+ return capacity + 1 + NumClonedBytes();
1005
1172
  }
1006
1173
 
1007
- // Given the capacity of a table, computes the offset (from the start of the
1008
- // backing allocation) at which the slots begin.
1009
- inline size_t SlotOffset(size_t capacity, size_t slot_align, bool has_infoz) {
1010
- assert(IsValidCapacity(capacity));
1011
- return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() +
1012
- slot_align - 1) &
1013
- (~slot_align + 1);
1174
+ // Computes the offset from the start of the backing allocation of control.
1175
+ // infoz and growth_info are stored at the beginning of the backing array.
1176
+ inline static size_t ControlOffset(bool has_infoz) {
1177
+ return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo);
1014
1178
  }
1015
1179
 
1016
- // Given the capacity of a table, computes the total size of the backing
1017
- // array.
1018
- inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align,
1019
- bool has_infoz) {
1020
- return SlotOffset(capacity, slot_align, has_infoz) + capacity * slot_size;
1021
- }
1180
+ // Helper class for computing offsets and allocation size of hash set fields.
1181
+ class RawHashSetLayout {
1182
+ public:
1183
+ explicit RawHashSetLayout(size_t capacity, size_t slot_align, bool has_infoz)
1184
+ : capacity_(capacity),
1185
+ control_offset_(ControlOffset(has_infoz)),
1186
+ generation_offset_(control_offset_ + NumControlBytes(capacity)),
1187
+ slot_offset_(
1188
+ (generation_offset_ + NumGenerationBytes() + slot_align - 1) &
1189
+ (~slot_align + 1)) {
1190
+ assert(IsValidCapacity(capacity));
1191
+ }
1192
+
1193
+ // Returns the capacity of a table.
1194
+ size_t capacity() const { return capacity_; }
1195
+
1196
+ // Returns precomputed offset from the start of the backing allocation of
1197
+ // control.
1198
+ size_t control_offset() const { return control_offset_; }
1199
+
1200
+ // Given the capacity of a table, computes the offset (from the start of the
1201
+ // backing allocation) of the generation counter (if it exists).
1202
+ size_t generation_offset() const { return generation_offset_; }
1203
+
1204
+ // Given the capacity of a table, computes the offset (from the start of the
1205
+ // backing allocation) at which the slots begin.
1206
+ size_t slot_offset() const { return slot_offset_; }
1207
+
1208
+ // Given the capacity of a table, computes the total size of the backing
1209
+ // array.
1210
+ size_t alloc_size(size_t slot_size) const {
1211
+ return slot_offset_ + capacity_ * slot_size;
1212
+ }
1213
+
1214
+ private:
1215
+ size_t capacity_;
1216
+ size_t control_offset_;
1217
+ size_t generation_offset_;
1218
+ size_t slot_offset_;
1219
+ };
1220
+
1221
+ struct HashtableFreeFunctionsAccess;
1222
+
1223
+ // We only allow a maximum of 1 SOO element, which makes the implementation
1224
+ // much simpler. Complications with multiple SOO elements include:
1225
+ // - Satisfying the guarantee that erasing one element doesn't invalidate
1226
+ // iterators to other elements means we would probably need actual SOO
1227
+ // control bytes.
1228
+ // - In order to prevent user code from depending on iteration order for small
1229
+ // tables, we would need to randomize the iteration order somehow.
1230
+ constexpr size_t SooCapacity() { return 1; }
1231
+ // Sentinel type to indicate SOO CommonFields construction.
1232
+ struct soo_tag_t {};
1233
+ // Sentinel type to indicate SOO CommonFields construction with full size.
1234
+ struct full_soo_tag_t {};
1235
+
1236
+ // Suppress erroneous uninitialized memory errors on GCC. For example, GCC
1237
+ // thinks that the call to slot_array() in find_or_prepare_insert() is reading
1238
+ // uninitialized memory, but slot_array is only called there when the table is
1239
+ // non-empty and this memory is initialized when the table is non-empty.
1240
+ #if !defined(__clang__) && defined(__GNUC__)
1241
+ #define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) \
1242
+ _Pragma("GCC diagnostic push") \
1243
+ _Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") \
1244
+ _Pragma("GCC diagnostic ignored \"-Wuninitialized\"") x; \
1245
+ _Pragma("GCC diagnostic pop")
1246
+ #define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) \
1247
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(return x)
1248
+ #else
1249
+ #define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) x
1250
+ #define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) return x
1251
+ #endif
1252
+
1253
+ // This allows us to work around an uninitialized memory warning when
1254
+ // constructing begin() iterators in empty hashtables.
1255
+ union MaybeInitializedPtr {
1256
+ void* get() const { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(p); }
1257
+ void set(void* ptr) { p = ptr; }
1258
+
1259
+ void* p;
1260
+ };
1261
+
1262
+ struct HeapPtrs {
1263
+ HeapPtrs() = default;
1264
+ explicit HeapPtrs(ctrl_t* c) : control(c) {}
1265
+
1266
+ // The control bytes (and, also, a pointer near to the base of the backing
1267
+ // array).
1268
+ //
1269
+ // This contains `capacity + 1 + NumClonedBytes()` entries, even
1270
+ // when the table is empty (hence EmptyGroup).
1271
+ //
1272
+ // Note that growth_info is stored immediately before this pointer.
1273
+ // May be uninitialized for SOO tables.
1274
+ ctrl_t* control;
1275
+
1276
+ // The beginning of the slots, located at `SlotOffset()` bytes after
1277
+ // `control`. May be uninitialized for empty tables.
1278
+ // Note: we can't use `slots` because Qt defines "slots" as a macro.
1279
+ MaybeInitializedPtr slot_array;
1280
+ };
1281
+
1282
+ // Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo
1283
+ // is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`.
1284
+ union HeapOrSoo {
1285
+ HeapOrSoo() = default;
1286
+ explicit HeapOrSoo(ctrl_t* c) : heap(c) {}
1287
+
1288
+ ctrl_t*& control() {
1289
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control);
1290
+ }
1291
+ ctrl_t* control() const {
1292
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control);
1293
+ }
1294
+ MaybeInitializedPtr& slot_array() {
1295
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array);
1296
+ }
1297
+ MaybeInitializedPtr slot_array() const {
1298
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array);
1299
+ }
1300
+ void* get_soo_data() {
1301
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data);
1302
+ }
1303
+ const void* get_soo_data() const {
1304
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data);
1305
+ }
1306
+
1307
+ HeapPtrs heap;
1308
+ unsigned char soo_data[sizeof(HeapPtrs)];
1309
+ };
1022
1310
 
1023
1311
  // CommonFields hold the fields in raw_hash_set that do not depend
1024
1312
  // on template parameters. This allows us to conveniently pass all
1025
1313
  // of this state to helper functions as a single argument.
1026
1314
  class CommonFields : public CommonFieldsGenerationInfo {
1027
1315
  public:
1028
- CommonFields() = default;
1316
+ CommonFields() : capacity_(0), size_(0), heap_or_soo_(EmptyGroup()) {}
1317
+ explicit CommonFields(soo_tag_t) : capacity_(SooCapacity()), size_(0) {}
1318
+ explicit CommonFields(full_soo_tag_t)
1319
+ : capacity_(SooCapacity()), size_(size_t{1} << HasInfozShift()) {}
1029
1320
 
1030
1321
  // Not copyable
1031
1322
  CommonFields(const CommonFields&) = delete;
@@ -1035,23 +1326,44 @@ class CommonFields : public CommonFieldsGenerationInfo {
1035
1326
  CommonFields(CommonFields&& that) = default;
1036
1327
  CommonFields& operator=(CommonFields&&) = default;
1037
1328
 
1038
- ctrl_t* control() const { return control_; }
1039
- void set_control(ctrl_t* c) { control_ = c; }
1329
+ template <bool kSooEnabled>
1330
+ static CommonFields CreateDefault() {
1331
+ return kSooEnabled ? CommonFields{soo_tag_t{}} : CommonFields{};
1332
+ }
1333
+
1334
+ // The inline data for SOO is written on top of control_/slots_.
1335
+ const void* soo_data() const { return heap_or_soo_.get_soo_data(); }
1336
+ void* soo_data() { return heap_or_soo_.get_soo_data(); }
1337
+
1338
+ HeapOrSoo heap_or_soo() const { return heap_or_soo_; }
1339
+ const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; }
1340
+
1341
+ ctrl_t* control() const { return heap_or_soo_.control(); }
1342
+ void set_control(ctrl_t* c) { heap_or_soo_.control() = c; }
1040
1343
  void* backing_array_start() const {
1041
- // growth_left (and maybe infoz) is stored before control bytes.
1344
+ // growth_info (and maybe infoz) is stored before control bytes.
1042
1345
  assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0);
1043
1346
  return control() - ControlOffset(has_infoz());
1044
1347
  }
1045
1348
 
1046
1349
  // Note: we can't use slots() because Qt defines "slots" as a macro.
1047
- void* slot_array() const { return slots_; }
1048
- void set_slots(void* s) { slots_ = s; }
1350
+ void* slot_array() const { return heap_or_soo_.slot_array().get(); }
1351
+ MaybeInitializedPtr slots_union() const { return heap_or_soo_.slot_array(); }
1352
+ void set_slots(void* s) { heap_or_soo_.slot_array().set(s); }
1049
1353
 
1050
1354
  // The number of filled slots.
1051
1355
  size_t size() const { return size_ >> HasInfozShift(); }
1052
1356
  void set_size(size_t s) {
1053
1357
  size_ = (s << HasInfozShift()) | (size_ & HasInfozMask());
1054
1358
  }
1359
+ void set_empty_soo() {
1360
+ AssertInSooMode();
1361
+ size_ = 0;
1362
+ }
1363
+ void set_full_soo() {
1364
+ AssertInSooMode();
1365
+ size_ = size_t{1} << HasInfozShift();
1366
+ }
1055
1367
  void increment_size() {
1056
1368
  assert(size() < capacity());
1057
1369
  size_ += size_t{1} << HasInfozShift();
@@ -1070,15 +1382,17 @@ class CommonFields : public CommonFieldsGenerationInfo {
1070
1382
 
1071
1383
  // The number of slots we can still fill without needing to rehash.
1072
1384
  // This is stored in the heap allocation before the control bytes.
1073
- size_t growth_left() const {
1074
- const size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1;
1075
- assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
1385
+ // TODO(b/289225379): experiment with moving growth_info back inline to
1386
+ // increase room for SOO.
1387
+ size_t growth_left() const { return growth_info().GetGrowthLeft(); }
1388
+
1389
+ GrowthInfo& growth_info() {
1390
+ auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1;
1391
+ assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0);
1076
1392
  return *gl_ptr;
1077
1393
  }
1078
- void set_growth_left(size_t gl) {
1079
- size_t* gl_ptr = reinterpret_cast<size_t*>(control()) - 1;
1080
- assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
1081
- *gl_ptr = gl;
1394
+ GrowthInfo growth_info() const {
1395
+ return const_cast<CommonFields*>(this)->growth_info();
1082
1396
  }
1083
1397
 
1084
1398
  bool has_infoz() const {
@@ -1103,12 +1417,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
1103
1417
  should_rehash_for_bug_detection_on_insert(control(), capacity());
1104
1418
  }
1105
1419
  bool should_rehash_for_bug_detection_on_move() const {
1106
- return CommonFieldsGenerationInfo::
1107
- should_rehash_for_bug_detection_on_move(control(), capacity());
1108
- }
1109
- void maybe_increment_generation_on_move() {
1110
- if (capacity() == 0) return;
1111
- increment_generation();
1420
+ return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move(
1421
+ control(), capacity());
1112
1422
  }
1113
1423
  void reset_reserved_growth(size_t reservation) {
1114
1424
  CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size());
@@ -1116,7 +1426,16 @@ class CommonFields : public CommonFieldsGenerationInfo {
1116
1426
 
1117
1427
  // The size of the backing array allocation.
1118
1428
  size_t alloc_size(size_t slot_size, size_t slot_align) const {
1119
- return AllocSize(capacity(), slot_size, slot_align, has_infoz());
1429
+ return RawHashSetLayout(capacity(), slot_align, has_infoz())
1430
+ .alloc_size(slot_size);
1431
+ }
1432
+
1433
+ // Move fields other than heap_or_soo_.
1434
+ void move_non_heap_or_soo_fields(CommonFields& that) {
1435
+ static_cast<CommonFieldsGenerationInfo&>(*this) =
1436
+ std::move(static_cast<CommonFieldsGenerationInfo&>(that));
1437
+ capacity_ = that.capacity_;
1438
+ size_ = that.size_;
1120
1439
  }
1121
1440
 
1122
1441
  // Returns the number of control bytes set to kDeleted. For testing only.
@@ -1132,21 +1451,12 @@ class CommonFields : public CommonFieldsGenerationInfo {
1132
1451
  return (size_t{1} << HasInfozShift()) - 1;
1133
1452
  }
1134
1453
 
1135
- // TODO(b/182800944): Investigate removing some of these fields:
1136
- // - control/slots can be derived from each other
1137
-
1138
- // The control bytes (and, also, a pointer near to the base of the backing
1139
- // array).
1140
- //
1141
- // This contains `capacity + 1 + NumClonedBytes()` entries, even
1142
- // when the table is empty (hence EmptyGroup).
1143
- //
1144
- // Note that growth_left is stored immediately before this pointer.
1145
- ctrl_t* control_ = EmptyGroup();
1146
-
1147
- // The beginning of the slots, located at `SlotOffset()` bytes after
1148
- // `control`. May be null for empty tables.
1149
- void* slots_ = nullptr;
1454
+ // We can't assert that SOO is enabled because we don't have SooEnabled(), but
1455
+ // we assert what we can.
1456
+ void AssertInSooMode() const {
1457
+ assert(capacity() == SooCapacity());
1458
+ assert(!has_infoz());
1459
+ }
1150
1460
 
1151
1461
  // The number of slots in the backing array. This is always 2^N-1 for an
1152
1462
  // integer N. NOTE: we tried experimenting with compressing the capacity and
@@ -1154,10 +1464,16 @@ class CommonFields : public CommonFieldsGenerationInfo {
1154
1464
  // power (N in 2^N-1), and (b) storing 2^N as the most significant bit of
1155
1465
  // size_ and storing size in the low bits. Both of these experiments were
1156
1466
  // regressions, presumably because we need capacity to do find operations.
1157
- size_t capacity_ = 0;
1467
+ size_t capacity_;
1158
1468
 
1159
1469
  // The size and also has one bit that stores whether we have infoz.
1160
- size_t size_ = 0;
1470
+ // TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_
1471
+ // encode the size in SOO case. We would be making size()/capacity() more
1472
+ // expensive in order to have more SOO space.
1473
+ size_t size_;
1474
+
1475
+ // Either the control/slots pointers or the SOO slot.
1476
+ HeapOrSoo heap_or_soo_;
1161
1477
  };
1162
1478
 
1163
1479
  template <class Policy, class Hash, class Eq, class Alloc>
@@ -1320,6 +1636,10 @@ inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a,
1320
1636
  const void* const& slot_b) {
1321
1637
  // If either control byte is null, then we can't tell.
1322
1638
  if (ctrl_a == nullptr || ctrl_b == nullptr) return true;
1639
+ const bool a_is_soo = IsSooControl(ctrl_a);
1640
+ if (a_is_soo != IsSooControl(ctrl_b)) return false;
1641
+ if (a_is_soo) return slot_a == slot_b;
1642
+
1323
1643
  const void* low_slot = slot_a;
1324
1644
  const void* hi_slot = slot_b;
1325
1645
  if (ctrl_a > ctrl_b) {
@@ -1343,41 +1663,45 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b,
1343
1663
  // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout
1344
1664
  // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve
1345
1665
  // the chances that the hot paths will be inlined.
1666
+
1667
+ // fail_if(is_invalid, message) crashes when is_invalid is true and provides
1668
+ // an error message based on `message`.
1669
+ const auto fail_if = [](bool is_invalid, const char* message) {
1670
+ if (ABSL_PREDICT_FALSE(is_invalid)) {
1671
+ ABSL_RAW_LOG(FATAL, "Invalid iterator comparison. %s", message);
1672
+ }
1673
+ };
1674
+
1346
1675
  const bool a_is_default = ctrl_a == EmptyGroup();
1347
1676
  const bool b_is_default = ctrl_b == EmptyGroup();
1348
- if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) {
1349
- ABSL_RAW_LOG(
1350
- FATAL,
1351
- "Invalid iterator comparison. Comparing default-constructed iterator "
1352
- "with non-default-constructed iterator.");
1353
- }
1354
1677
  if (a_is_default && b_is_default) return;
1678
+ fail_if(a_is_default != b_is_default,
1679
+ "Comparing default-constructed hashtable iterator with a "
1680
+ "non-default-constructed hashtable iterator.");
1355
1681
 
1356
1682
  if (SwisstableGenerationsEnabled()) {
1357
1683
  if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return;
1684
+ // Users don't need to know whether the tables are SOO so don't mention SOO
1685
+ // in the debug message.
1686
+ const bool a_is_soo = IsSooControl(ctrl_a);
1687
+ const bool b_is_soo = IsSooControl(ctrl_b);
1688
+ fail_if(a_is_soo != b_is_soo || (a_is_soo && b_is_soo),
1689
+ "Comparing iterators from different hashtables.");
1690
+
1358
1691
  const bool a_is_empty = IsEmptyGeneration(generation_ptr_a);
1359
1692
  const bool b_is_empty = IsEmptyGeneration(generation_ptr_b);
1360
- if (a_is_empty != b_is_empty) {
1361
- ABSL_RAW_LOG(FATAL,
1362
- "Invalid iterator comparison. Comparing iterator from a "
1363
- "non-empty hashtable with an iterator from an empty "
1364
- "hashtable.");
1365
- }
1366
- if (a_is_empty && b_is_empty) {
1367
- ABSL_RAW_LOG(FATAL,
1368
- "Invalid iterator comparison. Comparing iterators from "
1369
- "different empty hashtables.");
1370
- }
1693
+ fail_if(a_is_empty != b_is_empty,
1694
+ "Comparing an iterator from an empty hashtable with an iterator "
1695
+ "from a non-empty hashtable.");
1696
+ fail_if(a_is_empty && b_is_empty,
1697
+ "Comparing iterators from different empty hashtables.");
1698
+
1371
1699
  const bool a_is_end = ctrl_a == nullptr;
1372
1700
  const bool b_is_end = ctrl_b == nullptr;
1373
- if (a_is_end || b_is_end) {
1374
- ABSL_RAW_LOG(FATAL,
1375
- "Invalid iterator comparison. Comparing iterator with an "
1376
- "end() iterator from a different hashtable.");
1377
- }
1378
- ABSL_RAW_LOG(FATAL,
1379
- "Invalid iterator comparison. Comparing non-end() iterators "
1380
- "from different hashtables.");
1701
+ fail_if(a_is_end || b_is_end,
1702
+ "Comparing iterator with an end() iterator from a different "
1703
+ "hashtable.");
1704
+ fail_if(true, "Comparing non-end() iterators from different hashtables.");
1381
1705
  } else {
1382
1706
  ABSL_HARDENING_ASSERT(
1383
1707
  AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) &&
@@ -1432,20 +1756,17 @@ template <typename = void>
1432
1756
  inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
1433
1757
  auto seq = probe(common, hash);
1434
1758
  const ctrl_t* ctrl = common.control();
1759
+ if (IsEmptyOrDeleted(ctrl[seq.offset()]) &&
1760
+ !ShouldInsertBackwards(common.capacity(), hash, ctrl)) {
1761
+ return {seq.offset(), /*probe_length=*/0};
1762
+ }
1435
1763
  while (true) {
1436
- GroupEmptyOrDeleted g{ctrl + seq.offset()};
1764
+ GroupFullEmptyOrDeleted g{ctrl + seq.offset()};
1437
1765
  auto mask = g.MaskEmptyOrDeleted();
1438
1766
  if (mask) {
1439
- #if !defined(NDEBUG)
1440
- // We want to add entropy even when ASLR is not enabled.
1441
- // In debug build we will randomly insert in either the front or back of
1442
- // the group.
1443
- // TODO(kfm,sbenza): revisit after we do unconditional mixing
1444
- if (!is_small(common.capacity()) && ShouldInsertBackwards(hash, ctrl)) {
1445
- return {seq.offset(mask.HighestBitSet()), seq.index()};
1446
- }
1447
- #endif
1448
- return {seq.offset(mask.LowestBitSet()), seq.index()};
1767
+ return {
1768
+ seq.offset(GetInsertionOffset(mask, common.capacity(), hash, ctrl)),
1769
+ seq.index()};
1449
1770
  }
1450
1771
  seq.next();
1451
1772
  assert(seq.index() <= common.capacity() && "full table!");
@@ -1462,7 +1783,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t);
1462
1783
  FindInfo find_first_non_full_outofline(const CommonFields&, size_t);
1463
1784
 
1464
1785
  inline void ResetGrowthLeft(CommonFields& common) {
1465
- common.set_growth_left(CapacityToGrowth(common.capacity()) - common.size());
1786
+ common.growth_info().InitGrowthLeftNoDeleted(
1787
+ CapacityToGrowth(common.capacity()) - common.size());
1466
1788
  }
1467
1789
 
1468
1790
  // Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire
@@ -1476,43 +1798,140 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) {
1476
1798
  SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity);
1477
1799
  }
1478
1800
 
1479
- // Sets `ctrl[i]` to `h`.
1480
- //
1481
- // Unlike setting it directly, this function will perform bounds checks and
1482
- // mirror the value to the cloned tail if necessary.
1483
- inline void SetCtrl(const CommonFields& common, size_t i, ctrl_t h,
1484
- size_t slot_size) {
1485
- const size_t capacity = common.capacity();
1486
- assert(i < capacity);
1487
-
1488
- auto* slot_i = static_cast<const char*>(common.slot_array()) + i * slot_size;
1801
+ // Sets sanitizer poisoning for slot corresponding to control byte being set.
1802
+ inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h,
1803
+ size_t slot_size) {
1804
+ assert(i < c.capacity());
1805
+ auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size;
1489
1806
  if (IsFull(h)) {
1490
1807
  SanitizerUnpoisonMemoryRegion(slot_i, slot_size);
1491
1808
  } else {
1492
1809
  SanitizerPoisonMemoryRegion(slot_i, slot_size);
1493
1810
  }
1811
+ }
1494
1812
 
1495
- ctrl_t* ctrl = common.control();
1813
+ // Sets `ctrl[i]` to `h`.
1814
+ //
1815
+ // Unlike setting it directly, this function will perform bounds checks and
1816
+ // mirror the value to the cloned tail if necessary.
1817
+ inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h,
1818
+ size_t slot_size) {
1819
+ DoSanitizeOnSetCtrl(c, i, h, slot_size);
1820
+ ctrl_t* ctrl = c.control();
1496
1821
  ctrl[i] = h;
1497
- ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h;
1822
+ ctrl[((i - NumClonedBytes()) & c.capacity()) +
1823
+ (NumClonedBytes() & c.capacity())] = h;
1824
+ }
1825
+ // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
1826
+ inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) {
1827
+ SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size);
1498
1828
  }
1499
1829
 
1830
+ // Like SetCtrl, but in a single group table, we can save some operations when
1831
+ // setting the cloned control byte.
1832
+ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h,
1833
+ size_t slot_size) {
1834
+ assert(is_single_group(c.capacity()));
1835
+ DoSanitizeOnSetCtrl(c, i, h, slot_size);
1836
+ ctrl_t* ctrl = c.control();
1837
+ ctrl[i] = h;
1838
+ ctrl[i + c.capacity() + 1] = h;
1839
+ }
1500
1840
  // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
1501
- inline void SetCtrl(const CommonFields& common, size_t i, h2_t h,
1502
- size_t slot_size) {
1503
- SetCtrl(common, i, static_cast<ctrl_t>(h), slot_size);
1841
+ inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h,
1842
+ size_t slot_size) {
1843
+ SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size);
1504
1844
  }
1505
1845
 
1506
- // growth_left (which is a size_t) is stored with the backing array.
1846
+ // growth_info (which is a size_t) is stored with the backing array.
1507
1847
  constexpr size_t BackingArrayAlignment(size_t align_of_slot) {
1508
- return (std::max)(align_of_slot, alignof(size_t));
1848
+ return (std::max)(align_of_slot, alignof(GrowthInfo));
1509
1849
  }
1510
1850
 
1511
1851
  // Returns the address of the ith slot in slots where each slot occupies
1512
1852
  // slot_size.
1513
1853
  inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
1514
- return reinterpret_cast<void*>(reinterpret_cast<char*>(slot_array) +
1515
- (slot * slot_size));
1854
+ return static_cast<void*>(static_cast<char*>(slot_array) +
1855
+ (slot * slot_size));
1856
+ }
1857
+
1858
+ // Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`.
1859
+ // No insertion to the table allowed during Callback call.
1860
+ // Erasure is allowed only for the element passed to the callback.
1861
+ template <class SlotType, class Callback>
1862
+ ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots(
1863
+ const CommonFields& c, SlotType* slot, Callback cb) {
1864
+ const size_t cap = c.capacity();
1865
+ const ctrl_t* ctrl = c.control();
1866
+ if (is_small(cap)) {
1867
+ // Mirrored/cloned control bytes in small table are also located in the
1868
+ // first group (starting from position 0). We are taking group from position
1869
+ // `capacity` in order to avoid duplicates.
1870
+
1871
+ // Small tables capacity fits into portable group, where
1872
+ // GroupPortableImpl::MaskFull is more efficient for the
1873
+ // capacity <= GroupPortableImpl::kWidth.
1874
+ assert(cap <= GroupPortableImpl::kWidth &&
1875
+ "unexpectedly large small capacity");
1876
+ static_assert(Group::kWidth >= GroupPortableImpl::kWidth,
1877
+ "unexpected group width");
1878
+ // Group starts from kSentinel slot, so indices in the mask will
1879
+ // be increased by 1.
1880
+ const auto mask = GroupPortableImpl(ctrl + cap).MaskFull();
1881
+ --ctrl;
1882
+ --slot;
1883
+ for (uint32_t i : mask) {
1884
+ cb(ctrl + i, slot + i);
1885
+ }
1886
+ return;
1887
+ }
1888
+ size_t remaining = c.size();
1889
+ ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining;
1890
+ while (remaining != 0) {
1891
+ for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) {
1892
+ assert(IsFull(ctrl[i]) && "hash table was modified unexpectedly");
1893
+ cb(ctrl + i, slot + i);
1894
+ --remaining;
1895
+ }
1896
+ ctrl += Group::kWidth;
1897
+ slot += Group::kWidth;
1898
+ assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) &&
1899
+ "hash table was modified unexpectedly");
1900
+ }
1901
+ // NOTE: erasure of the current element is allowed in callback for
1902
+ // absl::erase_if specialization. So we use `>=`.
1903
+ assert(original_size_for_assert >= c.size() &&
1904
+ "hash table was modified unexpectedly");
1905
+ }
1906
+
1907
+ template <typename CharAlloc>
1908
+ constexpr bool ShouldSampleHashtablezInfo() {
1909
+ // Folks with custom allocators often make unwarranted assumptions about the
1910
+ // behavior of their classes vis-a-vis trivial destructability and what
1911
+ // calls they will or won't make. Avoid sampling for people with custom
1912
+ // allocators to get us out of this mess. This is not a hard guarantee but
1913
+ // a workaround while we plan the exact guarantee we want to provide.
1914
+ return std::is_same<CharAlloc, std::allocator<char>>::value;
1915
+ }
1916
+
1917
+ template <bool kSooEnabled>
1918
+ HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, size_t sizeof_key,
1919
+ size_t sizeof_value,
1920
+ size_t old_capacity, bool was_soo,
1921
+ HashtablezInfoHandle forced_infoz,
1922
+ CommonFields& c) {
1923
+ if (forced_infoz.IsSampled()) return forced_infoz;
1924
+ // In SOO, we sample on the first insertion so if this is an empty SOO case
1925
+ // (e.g. when reserve is called), then we still need to sample.
1926
+ if (kSooEnabled && was_soo && c.size() == 0) {
1927
+ return Sample(sizeof_slot, sizeof_key, sizeof_value, SooCapacity());
1928
+ }
1929
+ // For non-SOO cases, we sample whenever the capacity is increasing from zero
1930
+ // to non-zero.
1931
+ if (!kSooEnabled && old_capacity == 0) {
1932
+ return Sample(sizeof_slot, sizeof_key, sizeof_value, 0);
1933
+ }
1934
+ return c.infoz();
1516
1935
  }
1517
1936
 
1518
1937
  // Helper class to perform resize of the hash set.
@@ -1521,17 +1940,21 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
1521
1940
  // See GrowIntoSingleGroupShuffleControlBytes for details.
1522
1941
  class HashSetResizeHelper {
1523
1942
  public:
1524
- explicit HashSetResizeHelper(CommonFields& c)
1525
- : old_ctrl_(c.control()),
1526
- old_capacity_(c.capacity()),
1527
- had_infoz_(c.has_infoz()) {}
1528
-
1529
- // Optimized for small groups version of `find_first_non_full` applicable
1530
- // only right after calling `raw_hash_set::resize`.
1943
+ explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot,
1944
+ HashtablezInfoHandle forced_infoz)
1945
+ : old_capacity_(c.capacity()),
1946
+ had_infoz_(c.has_infoz()),
1947
+ was_soo_(was_soo),
1948
+ had_soo_slot_(had_soo_slot),
1949
+ forced_infoz_(forced_infoz) {}
1950
+
1951
+ // Optimized for small groups version of `find_first_non_full`.
1952
+ // Beneficial only right after calling `raw_hash_set::resize`.
1953
+ // It is safe to call in case capacity is big or was not changed, but there
1954
+ // will be no performance benefit.
1531
1955
  // It has implicit assumption that `resize` will call
1532
1956
  // `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`.
1533
- // Falls back to `find_first_non_full` in case of big groups, so it is
1534
- // safe to use after `rehash_and_grow_if_necessary`.
1957
+ // Falls back to `find_first_non_full` in case of big groups.
1535
1958
  static FindInfo FindFirstNonFullAfterResize(const CommonFields& c,
1536
1959
  size_t old_capacity,
1537
1960
  size_t hash) {
@@ -1553,14 +1976,30 @@ class HashSetResizeHelper {
1553
1976
  return FindInfo{offset, 0};
1554
1977
  }
1555
1978
 
1556
- ctrl_t* old_ctrl() const { return old_ctrl_; }
1979
+ HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; }
1980
+ void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); }
1981
+ ctrl_t* old_ctrl() const {
1982
+ assert(!was_soo_);
1983
+ return old_heap_or_soo_.control();
1984
+ }
1985
+ void* old_slots() const {
1986
+ assert(!was_soo_);
1987
+ return old_heap_or_soo_.slot_array().get();
1988
+ }
1557
1989
  size_t old_capacity() const { return old_capacity_; }
1558
1990
 
1991
+ // Returns the index of the SOO slot when growing from SOO to non-SOO in a
1992
+ // single group. See also InitControlBytesAfterSoo(). It's important to use
1993
+ // index 1 so that when resizing from capacity 1 to 3, we can still have
1994
+ // random iteration order between the first two inserted elements.
1995
+ // I.e. it allows inserting the second element at either index 0 or 2.
1996
+ static size_t SooSlotIndex() { return 1; }
1997
+
1559
1998
  // Allocates a backing array for the hashtable.
1560
1999
  // Reads `capacity` and updates all other fields based on the result of
1561
2000
  // the allocation.
1562
2001
  //
1563
- // It also may do the folowing actions:
2002
+ // It also may do the following actions:
1564
2003
  // 1. initialize control bytes
1565
2004
  // 2. initialize slots
1566
2005
  // 3. deallocate old slots.
@@ -1590,45 +2029,45 @@ class HashSetResizeHelper {
1590
2029
  //
1591
2030
  // Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation.
1592
2031
  template <typename Alloc, size_t SizeOfSlot, bool TransferUsesMemcpy,
1593
- size_t AlignOfSlot>
1594
- ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, void* old_slots,
1595
- Alloc alloc) {
2032
+ bool SooEnabled, size_t AlignOfSlot>
2033
+ ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc,
2034
+ ctrl_t soo_slot_h2,
2035
+ size_t key_size,
2036
+ size_t value_size) {
1596
2037
  assert(c.capacity());
1597
- // Folks with custom allocators often make unwarranted assumptions about the
1598
- // behavior of their classes vis-a-vis trivial destructability and what
1599
- // calls they will or won't make. Avoid sampling for people with custom
1600
- // allocators to get us out of this mess. This is not a hard guarantee but
1601
- // a workaround while we plan the exact guarantee we want to provide.
1602
- const size_t sample_size =
1603
- (std::is_same<Alloc, std::allocator<char>>::value &&
1604
- c.slot_array() == nullptr)
1605
- ? SizeOfSlot
1606
- : 0;
1607
2038
  HashtablezInfoHandle infoz =
1608
- sample_size > 0 ? Sample(sample_size) : c.infoz();
2039
+ ShouldSampleHashtablezInfo<Alloc>()
2040
+ ? SampleHashtablezInfo<SooEnabled>(SizeOfSlot, key_size, value_size,
2041
+ old_capacity_, was_soo_,
2042
+ forced_infoz_, c)
2043
+ : HashtablezInfoHandle{};
1609
2044
 
1610
2045
  const bool has_infoz = infoz.IsSampled();
1611
- const size_t cap = c.capacity();
1612
- const size_t alloc_size =
1613
- AllocSize(cap, SizeOfSlot, AlignOfSlot, has_infoz);
1614
- char* mem = static_cast<char*>(
1615
- Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size));
2046
+ RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz);
2047
+ char* mem = static_cast<char*>(Allocate<BackingArrayAlignment(AlignOfSlot)>(
2048
+ &alloc, layout.alloc_size(SizeOfSlot)));
1616
2049
  const GenerationType old_generation = c.generation();
1617
- c.set_generation_ptr(reinterpret_cast<GenerationType*>(
1618
- mem + GenerationOffset(cap, has_infoz)));
2050
+ c.set_generation_ptr(
2051
+ reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
1619
2052
  c.set_generation(NextGeneration(old_generation));
1620
- c.set_control(reinterpret_cast<ctrl_t*>(mem + ControlOffset(has_infoz)));
1621
- c.set_slots(mem + SlotOffset(cap, AlignOfSlot, has_infoz));
2053
+ c.set_control(reinterpret_cast<ctrl_t*>(mem + layout.control_offset()));
2054
+ c.set_slots(mem + layout.slot_offset());
1622
2055
  ResetGrowthLeft(c);
1623
2056
 
1624
2057
  const bool grow_single_group =
1625
- IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity());
1626
- if (old_capacity_ != 0 && grow_single_group) {
2058
+ IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity());
2059
+ if (SooEnabled && was_soo_ && grow_single_group) {
2060
+ InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity());
2061
+ if (TransferUsesMemcpy && had_soo_slot_) {
2062
+ TransferSlotAfterSoo(c, SizeOfSlot);
2063
+ }
2064
+ // SooEnabled implies that old_capacity_ != 0.
2065
+ } else if ((SooEnabled || old_capacity_ != 0) && grow_single_group) {
1627
2066
  if (TransferUsesMemcpy) {
1628
- GrowSizeIntoSingleGroupTransferable(c, old_slots, SizeOfSlot);
1629
- DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot, old_slots);
2067
+ GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot);
2068
+ DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot);
1630
2069
  } else {
1631
- GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity());
2070
+ GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity());
1632
2071
  }
1633
2072
  } else {
1634
2073
  ResetCtrl(c, SizeOfSlot);
@@ -1636,8 +2075,8 @@ class HashSetResizeHelper {
1636
2075
 
1637
2076
  c.set_has_infoz(has_infoz);
1638
2077
  if (has_infoz) {
1639
- infoz.RecordStorageChanged(c.size(), cap);
1640
- if (grow_single_group || old_capacity_ == 0) {
2078
+ infoz.RecordStorageChanged(c.size(), layout.capacity());
2079
+ if ((SooEnabled && was_soo_) || grow_single_group || old_capacity_ == 0) {
1641
2080
  infoz.RecordRehash(0);
1642
2081
  }
1643
2082
  c.set_infoz(infoz);
@@ -1651,21 +2090,22 @@ class HashSetResizeHelper {
1651
2090
  // PRECONDITIONS:
1652
2091
  // 1. GrowIntoSingleGroupShuffleControlBytes was already called.
1653
2092
  template <class PolicyTraits, class Alloc>
1654
- void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref,
1655
- typename PolicyTraits::slot_type* old_slots) {
2093
+ void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) {
1656
2094
  assert(old_capacity_ < Group::kWidth / 2);
1657
2095
  assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
1658
2096
  using slot_type = typename PolicyTraits::slot_type;
1659
2097
  assert(is_single_group(c.capacity()));
1660
2098
 
1661
- auto* new_slots = reinterpret_cast<slot_type*>(c.slot_array());
2099
+ auto* new_slots = static_cast<slot_type*>(c.slot_array());
2100
+ auto* old_slots_ptr = static_cast<slot_type*>(old_slots());
1662
2101
 
1663
2102
  size_t shuffle_bit = old_capacity_ / 2 + 1;
1664
2103
  for (size_t i = 0; i < old_capacity_; ++i) {
1665
- if (IsFull(old_ctrl_[i])) {
2104
+ if (IsFull(old_ctrl()[i])) {
1666
2105
  size_t new_i = i ^ shuffle_bit;
1667
2106
  SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type));
1668
- PolicyTraits::transfer(&alloc_ref, new_slots + new_i, old_slots + i);
2107
+ PolicyTraits::transfer(&alloc_ref, new_slots + new_i,
2108
+ old_slots_ptr + i);
1669
2109
  }
1670
2110
  }
1671
2111
  PoisonSingleGroupEmptySlots(c, sizeof(slot_type));
@@ -1673,11 +2113,12 @@ class HashSetResizeHelper {
1673
2113
 
1674
2114
  // Deallocates old backing array.
1675
2115
  template <size_t AlignOfSlot, class CharAlloc>
1676
- void DeallocateOld(CharAlloc alloc_ref, size_t slot_size, void* old_slots) {
1677
- SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_);
2116
+ void DeallocateOld(CharAlloc alloc_ref, size_t slot_size) {
2117
+ SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_);
2118
+ auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_);
1678
2119
  Deallocate<BackingArrayAlignment(AlignOfSlot)>(
1679
- &alloc_ref, old_ctrl_ - ControlOffset(had_infoz_),
1680
- AllocSize(old_capacity_, slot_size, AlignOfSlot, had_infoz_));
2120
+ &alloc_ref, old_ctrl() - layout.control_offset(),
2121
+ layout.alloc_size(slot_size));
1681
2122
  }
1682
2123
 
1683
2124
  private:
@@ -1692,8 +2133,12 @@ class HashSetResizeHelper {
1692
2133
  // Relocates control bytes and slots into new single group for
1693
2134
  // transferable objects.
1694
2135
  // Must be called only if IsGrowingIntoSingleGroupApplicable returned true.
1695
- void GrowSizeIntoSingleGroupTransferable(CommonFields& c, void* old_slots,
1696
- size_t slot_size);
2136
+ void GrowSizeIntoSingleGroupTransferable(CommonFields& c, size_t slot_size);
2137
+
2138
+ // If there was an SOO slot and slots are transferable, transfers the SOO slot
2139
+ // into the new heap allocation. Must be called only if
2140
+ // IsGrowingIntoSingleGroupApplicable returned true.
2141
+ void TransferSlotAfterSoo(CommonFields& c, size_t slot_size);
1697
2142
 
1698
2143
  // Shuffle control bits deterministically to the next capacity.
1699
2144
  // Returns offset for newly added element with given hash.
@@ -1726,6 +2171,13 @@ class HashSetResizeHelper {
1726
2171
  void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl,
1727
2172
  size_t new_capacity) const;
1728
2173
 
2174
+ // If the table was SOO, initializes new control bytes. `h2` is the control
2175
+ // byte corresponding to the full slot. Must be called only if
2176
+ // IsGrowingIntoSingleGroupApplicable returned true.
2177
+ // Requires: `had_soo_slot_ || h2 == ctrl_t::kEmpty`.
2178
+ void InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2,
2179
+ size_t new_capacity);
2180
+
1729
2181
  // Shuffle trivially transferable slots in the way consistent with
1730
2182
  // GrowIntoSingleGroupShuffleControlBytes.
1731
2183
  //
@@ -1739,8 +2191,7 @@ class HashSetResizeHelper {
1739
2191
  // 1. new_slots are transferred from old_slots_ consistent with
1740
2192
  // GrowIntoSingleGroupShuffleControlBytes.
1741
2193
  // 2. Empty new_slots are *not* poisoned.
1742
- void GrowIntoSingleGroupShuffleTransferableSlots(void* old_slots,
1743
- void* new_slots,
2194
+ void GrowIntoSingleGroupShuffleTransferableSlots(void* new_slots,
1744
2195
  size_t slot_size) const;
1745
2196
 
1746
2197
  // Poison empty slots that were transferred using the deterministic algorithm
@@ -1760,11 +2211,24 @@ class HashSetResizeHelper {
1760
2211
  }
1761
2212
  }
1762
2213
 
1763
- ctrl_t* old_ctrl_;
2214
+ HeapOrSoo old_heap_or_soo_;
1764
2215
  size_t old_capacity_;
1765
2216
  bool had_infoz_;
2217
+ bool was_soo_;
2218
+ bool had_soo_slot_;
2219
+ // Either null infoz or a pre-sampled forced infoz for SOO tables.
2220
+ HashtablezInfoHandle forced_infoz_;
1766
2221
  };
1767
2222
 
2223
+ inline void PrepareInsertCommon(CommonFields& common) {
2224
+ common.increment_size();
2225
+ common.maybe_increment_generation_on_insert();
2226
+ }
2227
+
2228
+ // Like prepare_insert, but for the case of inserting into a full SOO table.
2229
+ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size,
2230
+ CommonFields& common);
2231
+
1768
2232
  // PolicyFunctions bundles together some information for a particular
1769
2233
  // raw_hash_set<T, ...> instantiation. This information is passed to
1770
2234
  // type-erased functions that want to do small amounts of type-specific
@@ -1772,21 +2236,29 @@ class HashSetResizeHelper {
1772
2236
  struct PolicyFunctions {
1773
2237
  size_t slot_size;
1774
2238
 
2239
+ // Returns the pointer to the hash function stored in the set.
2240
+ const void* (*hash_fn)(const CommonFields& common);
2241
+
1775
2242
  // Returns the hash of the pointed-to slot.
1776
- size_t (*hash_slot)(void* set, void* slot);
2243
+ size_t (*hash_slot)(const void* hash_fn, void* slot);
1777
2244
 
1778
- // Transfer the contents of src_slot to dst_slot.
2245
+ // Transfers the contents of src_slot to dst_slot.
1779
2246
  void (*transfer)(void* set, void* dst_slot, void* src_slot);
1780
2247
 
1781
- // Deallocate the backing store from common.
2248
+ // Deallocates the backing store from common.
1782
2249
  void (*dealloc)(CommonFields& common, const PolicyFunctions& policy);
2250
+
2251
+ // Resizes set to the new capacity.
2252
+ // Arguments are used as in raw_hash_set::resize_impl.
2253
+ void (*resize)(CommonFields& common, size_t new_capacity,
2254
+ HashtablezInfoHandle forced_infoz);
1783
2255
  };
1784
2256
 
1785
2257
  // ClearBackingArray clears the backing array, either modifying it in place,
1786
2258
  // or creating a new one based on the value of "reuse".
1787
2259
  // REQUIRES: c.capacity > 0
1788
2260
  void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
1789
- bool reuse);
2261
+ bool reuse, bool soo_enabled);
1790
2262
 
1791
2263
  // Type-erased version of raw_hash_set::erase_meta_only.
1792
2264
  void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size);
@@ -1817,9 +2289,26 @@ ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) {
1817
2289
  memcpy(dst, src, SizeOfSlot);
1818
2290
  }
1819
2291
 
1820
- // Type-erased version of raw_hash_set::drop_deletes_without_resize.
1821
- void DropDeletesWithoutResize(CommonFields& common,
1822
- const PolicyFunctions& policy, void* tmp_space);
2292
+ // Type erased raw_hash_set::get_hash_ref_fn for the empty hash function case.
2293
+ const void* GetHashRefForEmptyHasher(const CommonFields& common);
2294
+
2295
+ // Given the hash of a value not currently in the table and the first empty
2296
+ // slot in the probe sequence, finds a viable slot index to insert it at.
2297
+ //
2298
+ // In case there's no space left, the table can be resized or rehashed
2299
+ // (for tables with deleted slots, see FindInsertPositionWithGrowthOrRehash).
2300
+ //
2301
+ // In the case of absence of deleted slots and positive growth_left, the element
2302
+ // can be inserted in the provided `target` position.
2303
+ //
2304
+ // When the table has deleted slots (according to GrowthInfo), the target
2305
+ // position will be searched one more time using `find_first_non_full`.
2306
+ //
2307
+ // REQUIRES: Table is not SOO.
2308
+ // REQUIRES: At least one non-full slot available.
2309
+ // REQUIRES: `target` is a valid empty position to insert.
2310
+ size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target,
2311
+ const PolicyFunctions& policy);
1823
2312
 
1824
2313
  // A SwissTable.
1825
2314
  //
@@ -1875,6 +2364,26 @@ class raw_hash_set {
1875
2364
  using key_arg = typename KeyArgImpl::template type<K, key_type>;
1876
2365
 
1877
2366
  private:
2367
+ // TODO(b/289225379): we could add extra SOO space inside raw_hash_set
2368
+ // after CommonFields to allow inlining larger slot_types (e.g. std::string),
2369
+ // but it's a bit complicated if we want to support incomplete mapped_type in
2370
+ // flat_hash_map. We could potentially do this for flat_hash_set and for an
2371
+ // allowlist of `mapped_type`s of flat_hash_map that includes e.g. arithmetic
2372
+ // types, strings, cords, and pairs/tuples of allowlisted types.
2373
+ constexpr static bool SooEnabled() {
2374
+ return PolicyTraits::soo_enabled() &&
2375
+ sizeof(slot_type) <= sizeof(HeapOrSoo) &&
2376
+ alignof(slot_type) <= alignof(HeapOrSoo);
2377
+ }
2378
+
2379
+ // Whether `size` fits in the SOO capacity of this table.
2380
+ bool fits_in_soo(size_t size) const {
2381
+ return SooEnabled() && size <= SooCapacity();
2382
+ }
2383
+ // Whether this table is in SOO mode or non-SOO mode.
2384
+ bool is_soo() const { return fits_in_soo(capacity()); }
2385
+ bool is_full_soo() const { return is_soo() && !empty(); }
2386
+
1878
2387
  // Give an early error when key_type is not hashable/eq.
1879
2388
  auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
1880
2389
  auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k));
@@ -1928,6 +2437,7 @@ class raw_hash_set {
1928
2437
 
1929
2438
  class iterator : private HashSetIteratorGenerationInfo {
1930
2439
  friend class raw_hash_set;
2440
+ friend struct HashtableFreeFunctionsAccess;
1931
2441
 
1932
2442
  public:
1933
2443
  using iterator_category = std::forward_iterator_tag;
@@ -1958,6 +2468,7 @@ class raw_hash_set {
1958
2468
  ++ctrl_;
1959
2469
  ++slot_;
1960
2470
  skip_empty_or_deleted();
2471
+ if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr;
1961
2472
  return *this;
1962
2473
  }
1963
2474
  // PRECONDITION: not an end() iterator.
@@ -1988,22 +2499,31 @@ class raw_hash_set {
1988
2499
  // not equal to any end iterator.
1989
2500
  ABSL_ASSUME(ctrl != nullptr);
1990
2501
  }
2502
+ // This constructor is used in begin() to avoid an MSan
2503
+ // use-of-uninitialized-value error. Delegating from this constructor to
2504
+ // the previous one doesn't avoid the error.
2505
+ iterator(ctrl_t* ctrl, MaybeInitializedPtr slot,
2506
+ const GenerationType* generation_ptr)
2507
+ : HashSetIteratorGenerationInfo(generation_ptr),
2508
+ ctrl_(ctrl),
2509
+ slot_(to_slot(slot.get())) {
2510
+ // This assumption helps the compiler know that any non-end iterator is
2511
+ // not equal to any end iterator.
2512
+ ABSL_ASSUME(ctrl != nullptr);
2513
+ }
1991
2514
  // For end() iterators.
1992
2515
  explicit iterator(const GenerationType* generation_ptr)
1993
2516
  : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {}
1994
2517
 
1995
- // Fixes up `ctrl_` to point to a full by advancing it and `slot_` until
1996
- // they reach one.
1997
- //
1998
- // If a sentinel is reached, we null `ctrl_` out instead.
2518
+ // Fixes up `ctrl_` to point to a full or sentinel by advancing `ctrl_` and
2519
+ // `slot_` until they reach one.
1999
2520
  void skip_empty_or_deleted() {
2000
2521
  while (IsEmptyOrDeleted(*ctrl_)) {
2001
2522
  uint32_t shift =
2002
- GroupEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted();
2523
+ GroupFullEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted();
2003
2524
  ctrl_ += shift;
2004
2525
  slot_ += shift;
2005
2526
  }
2006
- if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr;
2007
2527
  }
2008
2528
 
2009
2529
  ctrl_t* control() const { return ctrl_; }
@@ -2091,8 +2611,9 @@ class raw_hash_set {
2091
2611
  size_t bucket_count, const hasher& hash = hasher(),
2092
2612
  const key_equal& eq = key_equal(),
2093
2613
  const allocator_type& alloc = allocator_type())
2094
- : settings_(CommonFields{}, hash, eq, alloc) {
2095
- if (bucket_count) {
2614
+ : settings_(CommonFields::CreateDefault<SooEnabled()>(), hash, eq,
2615
+ alloc) {
2616
+ if (bucket_count > (SooEnabled() ? SooCapacity() : 0)) {
2096
2617
  resize(NormalizeCapacity(bucket_count));
2097
2618
  }
2098
2619
  }
@@ -2193,22 +2714,69 @@ class raw_hash_set {
2193
2714
  that.alloc_ref())) {}
2194
2715
 
2195
2716
  raw_hash_set(const raw_hash_set& that, const allocator_type& a)
2196
- : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) {
2717
+ : raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(),
2718
+ that.eq_ref(), a) {
2197
2719
  const size_t size = that.size();
2198
- if (size == 0) return;
2199
- reserve(size);
2200
- // Because the table is guaranteed to be empty, we can do something faster
2201
- // than a full `insert`.
2202
- for (const auto& v : that) {
2203
- const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v);
2204
- auto target = find_first_non_full_outofline(common(), hash);
2205
- SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type));
2206
- emplace_at(target.offset, v);
2207
- common().maybe_increment_generation_on_insert();
2208
- infoz().RecordInsert(hash, target.probe_length);
2720
+ if (size == 0) {
2721
+ return;
2722
+ }
2723
+ // We don't use `that.is_soo()` here because `that` can have non-SOO
2724
+ // capacity but have a size that fits into SOO capacity.
2725
+ if (fits_in_soo(size)) {
2726
+ assert(size == 1);
2727
+ common().set_full_soo();
2728
+ emplace_at(soo_iterator(), *that.begin());
2729
+ const HashtablezInfoHandle infoz = try_sample_soo();
2730
+ if (infoz.IsSampled()) resize_with_soo_infoz(infoz);
2731
+ return;
2732
+ }
2733
+ assert(!that.is_soo());
2734
+ const size_t cap = capacity();
2735
+ // Note about single group tables:
2736
+ // 1. It is correct to have any order of elements.
2737
+ // 2. Order has to be non deterministic.
2738
+ // 3. We are assigning elements with arbitrary `shift` starting from
2739
+ // `capacity + shift` position.
2740
+ // 4. `shift` must be coprime with `capacity + 1` in order to be able to use
2741
+ // modular arithmetic to traverse all positions, instead if cycling
2742
+ // through a subset of positions. Odd numbers are coprime with any
2743
+ // `capacity + 1` (2^N).
2744
+ size_t offset = cap;
2745
+ const size_t shift =
2746
+ is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0;
2747
+ IterateOverFullSlots(
2748
+ that.common(), that.slot_array(),
2749
+ [&](const ctrl_t* that_ctrl,
2750
+ slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE {
2751
+ if (shift == 0) {
2752
+ // Big tables case. Position must be searched via probing.
2753
+ // The table is guaranteed to be empty, so we can do faster than
2754
+ // a full `insert`.
2755
+ const size_t hash = PolicyTraits::apply(
2756
+ HashElement{hash_ref()}, PolicyTraits::element(that_slot));
2757
+ FindInfo target = find_first_non_full_outofline(common(), hash);
2758
+ infoz().RecordInsert(hash, target.probe_length);
2759
+ offset = target.offset;
2760
+ } else {
2761
+ // Small tables case. Next position is computed via shift.
2762
+ offset = (offset + shift) & cap;
2763
+ }
2764
+ const h2_t h2 = static_cast<h2_t>(*that_ctrl);
2765
+ assert( // We rely that hash is not changed for small tables.
2766
+ H2(PolicyTraits::apply(HashElement{hash_ref()},
2767
+ PolicyTraits::element(that_slot))) == h2 &&
2768
+ "hash function value changed unexpectedly during the copy");
2769
+ SetCtrl(common(), offset, h2, sizeof(slot_type));
2770
+ emplace_at(iterator_at(offset), PolicyTraits::element(that_slot));
2771
+ common().maybe_increment_generation_on_insert();
2772
+ });
2773
+ if (shift != 0) {
2774
+ // On small table copy we do not record individual inserts.
2775
+ // RecordInsert requires hash, but it is unknown for small tables.
2776
+ infoz().RecordStorageChanged(size, cap);
2209
2777
  }
2210
2778
  common().set_size(size);
2211
- set_growth_left(growth_left() - size);
2779
+ growth_info().OverwriteManyEmptyAsFull(size);
2212
2780
  }
2213
2781
 
2214
2782
  ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept(
@@ -2220,16 +2788,22 @@ class raw_hash_set {
2220
2788
  // would create a nullptr functor that cannot be called.
2221
2789
  // TODO(b/296061262): move instead of copying hash/eq/alloc.
2222
2790
  // Note: we avoid using exchange for better generated code.
2223
- settings_(std::move(that.common()), that.hash_ref(), that.eq_ref(),
2224
- that.alloc_ref()) {
2225
- that.common() = CommonFields{};
2791
+ settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo()
2792
+ ? std::move(that.common())
2793
+ : CommonFields{full_soo_tag_t{}},
2794
+ that.hash_ref(), that.eq_ref(), that.alloc_ref()) {
2795
+ if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) {
2796
+ transfer(soo_slot(), that.soo_slot());
2797
+ }
2798
+ that.common() = CommonFields::CreateDefault<SooEnabled()>();
2226
2799
  maybe_increment_generation_or_rehash_on_move();
2227
2800
  }
2228
2801
 
2229
2802
  raw_hash_set(raw_hash_set&& that, const allocator_type& a)
2230
- : settings_(CommonFields{}, that.hash_ref(), that.eq_ref(), a) {
2803
+ : settings_(CommonFields::CreateDefault<SooEnabled()>(), that.hash_ref(),
2804
+ that.eq_ref(), a) {
2231
2805
  if (a == that.alloc_ref()) {
2232
- std::swap(common(), that.common());
2806
+ swap_common(that);
2233
2807
  maybe_increment_generation_or_rehash_on_move();
2234
2808
  } else {
2235
2809
  move_elements_allocs_unequal(std::move(that));
@@ -2264,8 +2838,12 @@ class raw_hash_set {
2264
2838
  ~raw_hash_set() { destructor_impl(); }
2265
2839
 
2266
2840
  iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND {
2267
- auto it = iterator_at(0);
2841
+ if (ABSL_PREDICT_FALSE(empty())) return end();
2842
+ if (is_soo()) return soo_iterator();
2843
+ iterator it = {control(), common().slots_union(),
2844
+ common().generation_ptr()};
2268
2845
  it.skip_empty_or_deleted();
2846
+ assert(IsFull(*it.control()));
2269
2847
  return it;
2270
2848
  }
2271
2849
  iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND {
@@ -2285,7 +2863,14 @@ class raw_hash_set {
2285
2863
 
2286
2864
  bool empty() const { return !size(); }
2287
2865
  size_t size() const { return common().size(); }
2288
- size_t capacity() const { return common().capacity(); }
2866
+ size_t capacity() const {
2867
+ const size_t cap = common().capacity();
2868
+ // Compiler complains when using functions in assume so use local variables.
2869
+ ABSL_ATTRIBUTE_UNUSED static constexpr bool kEnabled = SooEnabled();
2870
+ ABSL_ATTRIBUTE_UNUSED static constexpr size_t kCapacity = SooCapacity();
2871
+ ABSL_ASSUME(!kEnabled || cap >= kCapacity);
2872
+ return cap;
2873
+ }
2289
2874
  size_t max_size() const { return (std::numeric_limits<size_t>::max)(); }
2290
2875
 
2291
2876
  ABSL_ATTRIBUTE_REINITIALIZES void clear() {
@@ -2299,9 +2884,13 @@ class raw_hash_set {
2299
2884
  const size_t cap = capacity();
2300
2885
  if (cap == 0) {
2301
2886
  // Already guaranteed to be empty; so nothing to do.
2887
+ } else if (is_soo()) {
2888
+ if (!empty()) destroy(soo_slot());
2889
+ common().set_empty_soo();
2302
2890
  } else {
2303
2891
  destroy_slots();
2304
- ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128);
2892
+ ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128,
2893
+ SooEnabled());
2305
2894
  }
2306
2895
  common().set_reserved_growth(0);
2307
2896
  common().set_reservation_size(0);
@@ -2432,7 +3021,7 @@ class raw_hash_set {
2432
3021
  std::pair<iterator, bool> emplace(Args&&... args)
2433
3022
  ABSL_ATTRIBUTE_LIFETIME_BOUND {
2434
3023
  alignas(slot_type) unsigned char raw[sizeof(slot_type)];
2435
- slot_type* slot = reinterpret_cast<slot_type*>(&raw);
3024
+ slot_type* slot = to_slot(&raw);
2436
3025
 
2437
3026
  construct(slot, std::forward<Args>(args)...);
2438
3027
  const auto& elem = PolicyTraits::element(slot);
@@ -2496,11 +3085,11 @@ class raw_hash_set {
2496
3085
  F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND {
2497
3086
  auto res = find_or_prepare_insert(key);
2498
3087
  if (res.second) {
2499
- slot_type* slot = slot_array() + res.first;
3088
+ slot_type* slot = res.first.slot();
2500
3089
  std::forward<F>(f)(constructor(&alloc_ref(), &slot));
2501
3090
  assert(!slot);
2502
3091
  }
2503
- return iterator_at(res.first);
3092
+ return res.first;
2504
3093
  }
2505
3094
 
2506
3095
  // Extension API: support for heterogeneous keys.
@@ -2524,7 +3113,7 @@ class raw_hash_set {
2524
3113
  // this method returns void to reduce algorithmic complexity to O(1). The
2525
3114
  // iterator is invalidated, so any increment should be done before calling
2526
3115
  // erase. In order to erase while iterating across a map, use the following
2527
- // idiom (which also works for standard containers):
3116
+ // idiom (which also works for some standard containers):
2528
3117
  //
2529
3118
  // for (auto it = m.begin(), end = m.end(); it != end;) {
2530
3119
  // // `erase()` will invalidate `it`, so advance `it` first.
@@ -2540,7 +3129,11 @@ class raw_hash_set {
2540
3129
  void erase(iterator it) {
2541
3130
  AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()");
2542
3131
  destroy(it.slot());
2543
- erase_meta_only(it);
3132
+ if (is_soo()) {
3133
+ common().set_empty_soo();
3134
+ } else {
3135
+ erase_meta_only(it);
3136
+ }
2544
3137
  }
2545
3138
 
2546
3139
  iterator erase(const_iterator first,
@@ -2548,12 +3141,19 @@ class raw_hash_set {
2548
3141
  // We check for empty first because ClearBackingArray requires that
2549
3142
  // capacity() > 0 as a precondition.
2550
3143
  if (empty()) return end();
3144
+ if (first == last) return last.inner_;
3145
+ if (is_soo()) {
3146
+ destroy(soo_slot());
3147
+ common().set_empty_soo();
3148
+ return end();
3149
+ }
2551
3150
  if (first == begin() && last == end()) {
2552
3151
  // TODO(ezb): we access control bytes in destroy_slots so it could make
2553
3152
  // sense to combine destroy_slots and ClearBackingArray to avoid cache
2554
3153
  // misses when the table is large. Note that we also do this in clear().
2555
3154
  destroy_slots();
2556
- ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true);
3155
+ ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true,
3156
+ SooEnabled());
2557
3157
  common().set_reserved_growth(common().reservation_size());
2558
3158
  return end();
2559
3159
  }
@@ -2568,13 +3168,21 @@ class raw_hash_set {
2568
3168
  template <typename H, typename E>
2569
3169
  void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT
2570
3170
  assert(this != &src);
3171
+ // Returns whether insertion took place.
3172
+ const auto insert_slot = [this](slot_type* src_slot) {
3173
+ return PolicyTraits::apply(InsertSlot<false>{*this, std::move(*src_slot)},
3174
+ PolicyTraits::element(src_slot))
3175
+ .second;
3176
+ };
3177
+
3178
+ if (src.is_soo()) {
3179
+ if (src.empty()) return;
3180
+ if (insert_slot(src.soo_slot())) src.common().set_empty_soo();
3181
+ return;
3182
+ }
2571
3183
  for (auto it = src.begin(), e = src.end(); it != e;) {
2572
3184
  auto next = std::next(it);
2573
- if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot())},
2574
- PolicyTraits::element(it.slot()))
2575
- .second) {
2576
- src.erase_meta_only(it);
2577
- }
3185
+ if (insert_slot(it.slot())) src.erase_meta_only(it);
2578
3186
  it = next;
2579
3187
  }
2580
3188
  }
@@ -2588,7 +3196,11 @@ class raw_hash_set {
2588
3196
  AssertIsFull(position.control(), position.inner_.generation(),
2589
3197
  position.inner_.generation_ptr(), "extract()");
2590
3198
  auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot());
2591
- erase_meta_only(position);
3199
+ if (is_soo()) {
3200
+ common().set_empty_soo();
3201
+ } else {
3202
+ erase_meta_only(position);
3203
+ }
2592
3204
  return node;
2593
3205
  }
2594
3206
 
@@ -2605,7 +3217,7 @@ class raw_hash_set {
2605
3217
  IsNoThrowSwappable<allocator_type>(
2606
3218
  typename AllocTraits::propagate_on_container_swap{})) {
2607
3219
  using std::swap;
2608
- swap(common(), that.common());
3220
+ swap_common(that);
2609
3221
  swap(hash_ref(), that.hash_ref());
2610
3222
  swap(eq_ref(), that.eq_ref());
2611
3223
  SwapAlloc(alloc_ref(), that.alloc_ref(),
@@ -2613,17 +3225,41 @@ class raw_hash_set {
2613
3225
  }
2614
3226
 
2615
3227
  void rehash(size_t n) {
2616
- if (n == 0 && capacity() == 0) return;
2617
- if (n == 0 && size() == 0) {
2618
- ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false);
2619
- return;
3228
+ const size_t cap = capacity();
3229
+ if (n == 0) {
3230
+ if (cap == 0 || is_soo()) return;
3231
+ if (empty()) {
3232
+ ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false,
3233
+ SooEnabled());
3234
+ return;
3235
+ }
3236
+ if (fits_in_soo(size())) {
3237
+ // When the table is already sampled, we keep it sampled.
3238
+ if (infoz().IsSampled()) {
3239
+ const size_t kInitialSampledCapacity = NextCapacity(SooCapacity());
3240
+ if (capacity() > kInitialSampledCapacity) {
3241
+ resize(kInitialSampledCapacity);
3242
+ }
3243
+ // This asserts that we didn't lose sampling coverage in `resize`.
3244
+ assert(infoz().IsSampled());
3245
+ return;
3246
+ }
3247
+ alignas(slot_type) unsigned char slot_space[sizeof(slot_type)];
3248
+ slot_type* tmp_slot = to_slot(slot_space);
3249
+ transfer(tmp_slot, begin().slot());
3250
+ ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false,
3251
+ SooEnabled());
3252
+ transfer(soo_slot(), tmp_slot);
3253
+ common().set_full_soo();
3254
+ return;
3255
+ }
2620
3256
  }
2621
3257
 
2622
3258
  // bitor is a faster way of doing `max` here. We will round up to the next
2623
3259
  // power-of-2-minus-1, so bitor is good enough.
2624
3260
  auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size()));
2625
3261
  // n == 0 unconditionally rehashes as per the standard.
2626
- if (n == 0 || m > capacity()) {
3262
+ if (n == 0 || m > cap) {
2627
3263
  resize(m);
2628
3264
 
2629
3265
  // This is after resize, to ensure that we have completed the allocation
@@ -2633,7 +3269,9 @@ class raw_hash_set {
2633
3269
  }
2634
3270
 
2635
3271
  void reserve(size_t n) {
2636
- if (n > size() + growth_left()) {
3272
+ const size_t max_size_before_growth =
3273
+ is_soo() ? SooCapacity() : size() + growth_left();
3274
+ if (n > max_size_before_growth) {
2637
3275
  size_t m = GrowthToLowerboundCapacity(n);
2638
3276
  resize(NormalizeCapacity(m));
2639
3277
 
@@ -2666,6 +3304,7 @@ class raw_hash_set {
2666
3304
  // specific benchmarks indicating its importance.
2667
3305
  template <class K = key_type>
2668
3306
  void prefetch(const key_arg<K>& key) const {
3307
+ if (SooEnabled() ? is_soo() : capacity() == 0) return;
2669
3308
  (void)key;
2670
3309
  // Avoid probing if we won't be able to prefetch the addresses received.
2671
3310
  #ifdef ABSL_HAVE_PREFETCH
@@ -2686,26 +3325,16 @@ class raw_hash_set {
2686
3325
  template <class K = key_type>
2687
3326
  iterator find(const key_arg<K>& key,
2688
3327
  size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND {
2689
- auto seq = probe(common(), hash);
2690
- slot_type* slot_ptr = slot_array();
2691
- const ctrl_t* ctrl = control();
2692
- while (true) {
2693
- Group g{ctrl + seq.offset()};
2694
- for (uint32_t i : g.Match(H2(hash))) {
2695
- if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
2696
- EqualElement<K>{key, eq_ref()},
2697
- PolicyTraits::element(slot_ptr + seq.offset(i)))))
2698
- return iterator_at(seq.offset(i));
2699
- }
2700
- if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
2701
- seq.next();
2702
- assert(seq.index() <= capacity() && "full table!");
2703
- }
3328
+ AssertHashEqConsistent(key);
3329
+ if (is_soo()) return find_soo(key);
3330
+ return find_non_soo(key, hash);
2704
3331
  }
2705
3332
  template <class K = key_type>
2706
3333
  iterator find(const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND {
3334
+ AssertHashEqConsistent(key);
3335
+ if (is_soo()) return find_soo(key);
2707
3336
  prefetch_heap_block();
2708
- return find(key, hash_ref()(key));
3337
+ return find_non_soo(key, hash_ref()(key));
2709
3338
  }
2710
3339
 
2711
3340
  template <class K = key_type>
@@ -2716,8 +3345,7 @@ class raw_hash_set {
2716
3345
  template <class K = key_type>
2717
3346
  const_iterator find(const key_arg<K>& key) const
2718
3347
  ABSL_ATTRIBUTE_LIFETIME_BOUND {
2719
- prefetch_heap_block();
2720
- return find(key, hash_ref()(key));
3348
+ return const_cast<raw_hash_set*>(this)->find(key);
2721
3349
  }
2722
3350
 
2723
3351
  template <class K = key_type>
@@ -2791,6 +3419,8 @@ class raw_hash_set {
2791
3419
  friend struct absl::container_internal::hashtable_debug_internal::
2792
3420
  HashtableDebugAccess;
2793
3421
 
3422
+ friend struct absl::container_internal::HashtableFreeFunctionsAccess;
3423
+
2794
3424
  struct FindElement {
2795
3425
  template <class K, class... Args>
2796
3426
  const_iterator operator()(const K& key, Args&&...) const {
@@ -2824,7 +3454,7 @@ class raw_hash_set {
2824
3454
  if (res.second) {
2825
3455
  s.emplace_at(res.first, std::forward<Args>(args)...);
2826
3456
  }
2827
- return {s.iterator_at(res.first), res.second};
3457
+ return res;
2828
3458
  }
2829
3459
  raw_hash_set& s;
2830
3460
  };
@@ -2835,11 +3465,11 @@ class raw_hash_set {
2835
3465
  std::pair<iterator, bool> operator()(const K& key, Args&&...) && {
2836
3466
  auto res = s.find_or_prepare_insert(key);
2837
3467
  if (res.second) {
2838
- s.transfer(s.slot_array() + res.first, &slot);
3468
+ s.transfer(res.first.slot(), &slot);
2839
3469
  } else if (do_destroy) {
2840
3470
  s.destroy(&slot);
2841
3471
  }
2842
- return {s.iterator_at(res.first), res.second};
3472
+ return res;
2843
3473
  }
2844
3474
  raw_hash_set& s;
2845
3475
  // Constructed slot. Either moved into place or destroyed.
@@ -2858,17 +3488,55 @@ class raw_hash_set {
2858
3488
  PolicyTraits::transfer(&alloc_ref(), to, from);
2859
3489
  }
2860
3490
 
2861
- inline void destroy_slots() {
2862
- const size_t cap = capacity();
3491
+ // TODO(b/289225379): consider having a helper class that has the impls for
3492
+ // SOO functionality.
3493
+ template <class K = key_type>
3494
+ iterator find_soo(const key_arg<K>& key) {
3495
+ assert(is_soo());
3496
+ return empty() || !PolicyTraits::apply(EqualElement<K>{key, eq_ref()},
3497
+ PolicyTraits::element(soo_slot()))
3498
+ ? end()
3499
+ : soo_iterator();
3500
+ }
3501
+
3502
+ template <class K = key_type>
3503
+ iterator find_non_soo(const key_arg<K>& key, size_t hash) {
3504
+ assert(!is_soo());
3505
+ auto seq = probe(common(), hash);
2863
3506
  const ctrl_t* ctrl = control();
2864
- slot_type* slot = slot_array();
2865
- for (size_t i = 0; i != cap; ++i) {
2866
- if (IsFull(ctrl[i])) {
2867
- destroy(slot + i);
3507
+ while (true) {
3508
+ Group g{ctrl + seq.offset()};
3509
+ for (uint32_t i : g.Match(H2(hash))) {
3510
+ if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
3511
+ EqualElement<K>{key, eq_ref()},
3512
+ PolicyTraits::element(slot_array() + seq.offset(i)))))
3513
+ return iterator_at(seq.offset(i));
2868
3514
  }
3515
+ if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
3516
+ seq.next();
3517
+ assert(seq.index() <= capacity() && "full table!");
2869
3518
  }
2870
3519
  }
2871
3520
 
3521
+ // Conditionally samples hashtablez for SOO tables. This should be called on
3522
+ // insertion into an empty SOO table and in copy construction when the size
3523
+ // can fit in SOO capacity.
3524
+ inline HashtablezInfoHandle try_sample_soo() {
3525
+ assert(is_soo());
3526
+ if (!ShouldSampleHashtablezInfo<CharAlloc>()) return HashtablezInfoHandle{};
3527
+ return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type),
3528
+ SooCapacity());
3529
+ }
3530
+
3531
+ inline void destroy_slots() {
3532
+ assert(!is_soo());
3533
+ if (PolicyTraits::template destroy_is_trivial<Alloc>()) return;
3534
+ IterateOverFullSlots(
3535
+ common(), slot_array(),
3536
+ [&](const ctrl_t*, slot_type* slot)
3537
+ ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); });
3538
+ }
3539
+
2872
3540
  inline void dealloc() {
2873
3541
  assert(capacity() != 0);
2874
3542
  // Unpoison before returning the memory to the allocator.
@@ -2881,6 +3549,12 @@ class raw_hash_set {
2881
3549
 
2882
3550
  inline void destructor_impl() {
2883
3551
  if (capacity() == 0) return;
3552
+ if (is_soo()) {
3553
+ if (!empty()) {
3554
+ ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(destroy(soo_slot()));
3555
+ }
3556
+ return;
3557
+ }
2884
3558
  destroy_slots();
2885
3559
  dealloc();
2886
3560
  }
@@ -2890,10 +3564,16 @@ class raw_hash_set {
2890
3564
  // This merely updates the pertinent control byte. This can be used in
2891
3565
  // conjunction with Policy::transfer to move the object to another place.
2892
3566
  void erase_meta_only(const_iterator it) {
3567
+ assert(!is_soo());
2893
3568
  EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()),
2894
3569
  sizeof(slot_type));
2895
3570
  }
2896
3571
 
3572
+ size_t hash_of(slot_type* slot) const {
3573
+ return PolicyTraits::apply(HashElement{hash_ref()},
3574
+ PolicyTraits::element(slot));
3575
+ }
3576
+
2897
3577
  // Resizes table to the new capacity and move all elements to the new
2898
3578
  // positions accordingly.
2899
3579
  //
@@ -2902,143 +3582,165 @@ class raw_hash_set {
2902
3582
  // HashSetResizeHelper::FindFirstNonFullAfterResize(
2903
3583
  // common(), old_capacity, hash)
2904
3584
  // can be called right after `resize`.
2905
- ABSL_ATTRIBUTE_NOINLINE void resize(size_t new_capacity) {
3585
+ void resize(size_t new_capacity) {
3586
+ raw_hash_set::resize_impl(common(), new_capacity, HashtablezInfoHandle{});
3587
+ }
3588
+
3589
+ // As above, except that we also accept a pre-sampled, forced infoz for
3590
+ // SOO tables, since they need to switch from SOO to heap in order to
3591
+ // store the infoz.
3592
+ void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) {
3593
+ assert(forced_infoz.IsSampled());
3594
+ raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()),
3595
+ forced_infoz);
3596
+ }
3597
+
3598
+ // Resizes set to the new capacity.
3599
+ // It is a static function in order to use its pointer in GetPolicyFunctions.
3600
+ ABSL_ATTRIBUTE_NOINLINE static void resize_impl(
3601
+ CommonFields& common, size_t new_capacity,
3602
+ HashtablezInfoHandle forced_infoz) {
3603
+ raw_hash_set* set = reinterpret_cast<raw_hash_set*>(&common);
2906
3604
  assert(IsValidCapacity(new_capacity));
2907
- HashSetResizeHelper resize_helper(common());
2908
- auto* old_slots = slot_array();
2909
- common().set_capacity(new_capacity);
3605
+ assert(!set->fits_in_soo(new_capacity));
3606
+ const bool was_soo = set->is_soo();
3607
+ const bool had_soo_slot = was_soo && !set->empty();
3608
+ const ctrl_t soo_slot_h2 =
3609
+ had_soo_slot ? static_cast<ctrl_t>(H2(set->hash_of(set->soo_slot())))
3610
+ : ctrl_t::kEmpty;
3611
+ HashSetResizeHelper resize_helper(common, was_soo, had_soo_slot,
3612
+ forced_infoz);
3613
+ // Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in
3614
+ // HashSetResizeHelper constructor because it can't transfer slots when
3615
+ // transfer_uses_memcpy is false.
3616
+ // TODO(b/289225379): try to handle more of the SOO cases inside
3617
+ // InitializeSlots. See comment on cl/555990034 snapshot #63.
3618
+ if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) {
3619
+ resize_helper.old_heap_or_soo() = common.heap_or_soo();
3620
+ } else {
3621
+ set->transfer(set->to_slot(resize_helper.old_soo_data()),
3622
+ set->soo_slot());
3623
+ }
3624
+ common.set_capacity(new_capacity);
2910
3625
  // Note that `InitializeSlots` does different number initialization steps
2911
3626
  // depending on the values of `transfer_uses_memcpy` and capacities.
2912
3627
  // Refer to the comment in `InitializeSlots` for more details.
2913
3628
  const bool grow_single_group =
2914
3629
  resize_helper.InitializeSlots<CharAlloc, sizeof(slot_type),
2915
3630
  PolicyTraits::transfer_uses_memcpy(),
2916
- alignof(slot_type)>(
2917
- common(), const_cast<std::remove_const_t<slot_type>*>(old_slots),
2918
- CharAlloc(alloc_ref()));
3631
+ SooEnabled(), alignof(slot_type)>(
3632
+ common, CharAlloc(set->alloc_ref()), soo_slot_h2, sizeof(key_type),
3633
+ sizeof(value_type));
2919
3634
 
2920
- if (resize_helper.old_capacity() == 0) {
3635
+ // In the SooEnabled() case, capacity is never 0 so we don't check.
3636
+ if (!SooEnabled() && resize_helper.old_capacity() == 0) {
2921
3637
  // InitializeSlots did all the work including infoz().RecordRehash().
2922
3638
  return;
2923
3639
  }
3640
+ assert(resize_helper.old_capacity() > 0);
3641
+ // Nothing more to do in this case.
3642
+ if (was_soo && !had_soo_slot) return;
2924
3643
 
3644
+ slot_type* new_slots = set->slot_array();
2925
3645
  if (grow_single_group) {
2926
3646
  if (PolicyTraits::transfer_uses_memcpy()) {
2927
3647
  // InitializeSlots did all the work.
2928
3648
  return;
2929
3649
  }
2930
- // We want GrowSizeIntoSingleGroup to be called here in order to make
2931
- // InitializeSlots not depend on PolicyTraits.
2932
- resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common(), alloc_ref(),
2933
- old_slots);
3650
+ if (was_soo) {
3651
+ set->transfer(new_slots + resize_helper.SooSlotIndex(),
3652
+ to_slot(resize_helper.old_soo_data()));
3653
+ return;
3654
+ } else {
3655
+ // We want GrowSizeIntoSingleGroup to be called here in order to make
3656
+ // InitializeSlots not depend on PolicyTraits.
3657
+ resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common,
3658
+ set->alloc_ref());
3659
+ }
2934
3660
  } else {
2935
3661
  // InitializeSlots prepares control bytes to correspond to empty table.
2936
- auto* new_slots = slot_array();
2937
- size_t total_probe_length = 0;
2938
- for (size_t i = 0; i != resize_helper.old_capacity(); ++i) {
2939
- if (IsFull(resize_helper.old_ctrl()[i])) {
2940
- size_t hash = PolicyTraits::apply(
2941
- HashElement{hash_ref()}, PolicyTraits::element(old_slots + i));
2942
- auto target = find_first_non_full(common(), hash);
2943
- size_t new_i = target.offset;
2944
- total_probe_length += target.probe_length;
2945
- SetCtrl(common(), new_i, H2(hash), sizeof(slot_type));
2946
- transfer(new_slots + new_i, old_slots + i);
3662
+ const auto insert_slot = [&](slot_type* slot) {
3663
+ size_t hash = PolicyTraits::apply(HashElement{set->hash_ref()},
3664
+ PolicyTraits::element(slot));
3665
+ auto target = find_first_non_full(common, hash);
3666
+ SetCtrl(common, target.offset, H2(hash), sizeof(slot_type));
3667
+ set->transfer(new_slots + target.offset, slot);
3668
+ return target.probe_length;
3669
+ };
3670
+ if (was_soo) {
3671
+ insert_slot(to_slot(resize_helper.old_soo_data()));
3672
+ return;
3673
+ } else {
3674
+ auto* old_slots = static_cast<slot_type*>(resize_helper.old_slots());
3675
+ size_t total_probe_length = 0;
3676
+ for (size_t i = 0; i != resize_helper.old_capacity(); ++i) {
3677
+ if (IsFull(resize_helper.old_ctrl()[i])) {
3678
+ total_probe_length += insert_slot(old_slots + i);
3679
+ }
2947
3680
  }
3681
+ common.infoz().RecordRehash(total_probe_length);
2948
3682
  }
2949
- infoz().RecordRehash(total_probe_length);
2950
3683
  }
2951
- resize_helper.DeallocateOld<alignof(slot_type)>(
2952
- CharAlloc(alloc_ref()), sizeof(slot_type),
2953
- const_cast<std::remove_const_t<slot_type>*>(old_slots));
3684
+ resize_helper.DeallocateOld<alignof(slot_type)>(CharAlloc(set->alloc_ref()),
3685
+ sizeof(slot_type));
2954
3686
  }
2955
3687
 
2956
- // Prunes control bytes to remove as many tombstones as possible.
2957
- //
2958
- // See the comment on `rehash_and_grow_if_necessary()`.
2959
- inline void drop_deletes_without_resize() {
2960
- // Stack-allocate space for swapping elements.
2961
- alignas(slot_type) unsigned char tmp[sizeof(slot_type)];
2962
- DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp);
2963
- }
3688
+ // Casting directly from e.g. char* to slot_type* can cause compilation errors
3689
+ // on objective-C. This function converts to void* first, avoiding the issue.
3690
+ static slot_type* to_slot(void* buf) { return static_cast<slot_type*>(buf); }
2964
3691
 
2965
- // Called whenever the table *might* need to conditionally grow.
2966
- //
2967
- // This function is an optimization opportunity to perform a rehash even when
2968
- // growth is unnecessary, because vacating tombstones is beneficial for
2969
- // performance in the long-run.
2970
- void rehash_and_grow_if_necessary() {
2971
- const size_t cap = capacity();
2972
- if (cap > Group::kWidth &&
2973
- // Do these calculations in 64-bit to avoid overflow.
2974
- size() * uint64_t{32} <= cap * uint64_t{25}) {
2975
- // Squash DELETED without growing if there is enough capacity.
2976
- //
2977
- // Rehash in place if the current size is <= 25/32 of capacity.
2978
- // Rationale for such a high factor: 1) drop_deletes_without_resize() is
2979
- // faster than resize, and 2) it takes quite a bit of work to add
2980
- // tombstones. In the worst case, seems to take approximately 4
2981
- // insert/erase pairs to create a single tombstone and so if we are
2982
- // rehashing because of tombstones, we can afford to rehash-in-place as
2983
- // long as we are reclaiming at least 1/8 the capacity without doing more
2984
- // than 2X the work. (Where "work" is defined to be size() for rehashing
2985
- // or rehashing in place, and 1 for an insert or erase.) But rehashing in
2986
- // place is faster per operation than inserting or even doubling the size
2987
- // of the table, so we actually afford to reclaim even less space from a
2988
- // resize-in-place. The decision is to rehash in place if we can reclaim
2989
- // at about 1/8th of the usable capacity (specifically 3/28 of the
2990
- // capacity) which means that the total cost of rehashing will be a small
2991
- // fraction of the total work.
2992
- //
2993
- // Here is output of an experiment using the BM_CacheInSteadyState
2994
- // benchmark running the old case (where we rehash-in-place only if we can
2995
- // reclaim at least 7/16*capacity) vs. this code (which rehashes in place
2996
- // if we can recover 3/32*capacity).
2997
- //
2998
- // Note that although in the worst-case number of rehashes jumped up from
2999
- // 15 to 190, but the number of operations per second is almost the same.
3000
- //
3001
- // Abridged output of running BM_CacheInSteadyState benchmark from
3002
- // raw_hash_set_benchmark. N is the number of insert/erase operations.
3003
- //
3004
- // | OLD (recover >= 7/16 | NEW (recover >= 3/32)
3005
- // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes
3006
- // 448 | 145284 0.44 18 | 140118 0.44 19
3007
- // 493 | 152546 0.24 11 | 151417 0.48 28
3008
- // 538 | 151439 0.26 11 | 151152 0.53 38
3009
- // 583 | 151765 0.28 11 | 150572 0.57 50
3010
- // 628 | 150241 0.31 11 | 150853 0.61 66
3011
- // 672 | 149602 0.33 12 | 150110 0.66 90
3012
- // 717 | 149998 0.35 12 | 149531 0.70 129
3013
- // 762 | 149836 0.37 13 | 148559 0.74 190
3014
- // 807 | 149736 0.39 14 | 151107 0.39 14
3015
- // 852 | 150204 0.42 15 | 151019 0.42 15
3016
- drop_deletes_without_resize();
3692
+ // Requires that lhs does not have a full SOO slot.
3693
+ static void move_common(bool that_is_full_soo, allocator_type& rhs_alloc,
3694
+ CommonFields& lhs, CommonFields&& rhs) {
3695
+ if (PolicyTraits::transfer_uses_memcpy() || !that_is_full_soo) {
3696
+ lhs = std::move(rhs);
3017
3697
  } else {
3018
- // Otherwise grow the container.
3019
- resize(NextCapacity(cap));
3698
+ lhs.move_non_heap_or_soo_fields(rhs);
3699
+ // TODO(b/303305702): add reentrancy guard.
3700
+ PolicyTraits::transfer(&rhs_alloc, to_slot(lhs.soo_data()),
3701
+ to_slot(rhs.soo_data()));
3020
3702
  }
3021
3703
  }
3022
3704
 
3705
+ // Swaps common fields making sure to avoid memcpy'ing a full SOO slot if we
3706
+ // aren't allowed to do so.
3707
+ void swap_common(raw_hash_set& that) {
3708
+ using std::swap;
3709
+ if (PolicyTraits::transfer_uses_memcpy()) {
3710
+ swap(common(), that.common());
3711
+ return;
3712
+ }
3713
+ CommonFields tmp = CommonFields::CreateDefault<SooEnabled()>();
3714
+ const bool that_is_full_soo = that.is_full_soo();
3715
+ move_common(that_is_full_soo, that.alloc_ref(), tmp,
3716
+ std::move(that.common()));
3717
+ move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common()));
3718
+ move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp));
3719
+ }
3720
+
3023
3721
  void maybe_increment_generation_or_rehash_on_move() {
3024
- common().maybe_increment_generation_on_move();
3722
+ if (!SwisstableGenerationsEnabled() || capacity() == 0 || is_soo()) {
3723
+ return;
3724
+ }
3725
+ common().increment_generation();
3025
3726
  if (!empty() && common().should_rehash_for_bug_detection_on_move()) {
3026
3727
  resize(capacity());
3027
3728
  }
3028
3729
  }
3029
3730
 
3030
- template<bool propagate_alloc>
3731
+ template <bool propagate_alloc>
3031
3732
  raw_hash_set& assign_impl(raw_hash_set&& that) {
3032
3733
  // We don't bother checking for this/that aliasing. We just need to avoid
3033
3734
  // breaking the invariants in that case.
3034
3735
  destructor_impl();
3035
- common() = std::move(that.common());
3736
+ move_common(that.is_full_soo(), that.alloc_ref(), common(),
3737
+ std::move(that.common()));
3036
3738
  // TODO(b/296061262): move instead of copying hash/eq/alloc.
3037
3739
  hash_ref() = that.hash_ref();
3038
3740
  eq_ref() = that.eq_ref();
3039
3741
  CopyAlloc(alloc_ref(), that.alloc_ref(),
3040
3742
  std::integral_constant<bool, propagate_alloc>());
3041
- that.common() = CommonFields{};
3743
+ that.common() = CommonFields::CreateDefault<SooEnabled()>();
3042
3744
  maybe_increment_generation_or_rehash_on_move();
3043
3745
  return *this;
3044
3746
  }
@@ -3051,8 +3753,8 @@ class raw_hash_set {
3051
3753
  insert(std::move(PolicyTraits::element(it.slot())));
3052
3754
  that.destroy(it.slot());
3053
3755
  }
3054
- that.dealloc();
3055
- that.common() = CommonFields{};
3756
+ if (!that.is_soo()) that.dealloc();
3757
+ that.common() = CommonFields::CreateDefault<SooEnabled()>();
3056
3758
  maybe_increment_generation_or_rehash_on_move();
3057
3759
  return *this;
3058
3760
  }
@@ -3078,12 +3780,30 @@ class raw_hash_set {
3078
3780
  return move_elements_allocs_unequal(std::move(that));
3079
3781
  }
3080
3782
 
3081
- protected:
3082
- // Attempts to find `key` in the table; if it isn't found, returns a slot that
3083
- // the value can be inserted into, with the control byte already set to
3084
- // `key`'s H2.
3085
3783
  template <class K>
3086
- std::pair<size_t, bool> find_or_prepare_insert(const K& key) {
3784
+ std::pair<iterator, bool> find_or_prepare_insert_soo(const K& key) {
3785
+ if (empty()) {
3786
+ const HashtablezInfoHandle infoz = try_sample_soo();
3787
+ if (infoz.IsSampled()) {
3788
+ resize_with_soo_infoz(infoz);
3789
+ } else {
3790
+ common().set_full_soo();
3791
+ return {soo_iterator(), true};
3792
+ }
3793
+ } else if (PolicyTraits::apply(EqualElement<K>{key, eq_ref()},
3794
+ PolicyTraits::element(soo_slot()))) {
3795
+ return {soo_iterator(), false};
3796
+ } else {
3797
+ resize(NextCapacity(SooCapacity()));
3798
+ }
3799
+ const size_t index =
3800
+ PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common());
3801
+ return {iterator_at(index), true};
3802
+ }
3803
+
3804
+ template <class K>
3805
+ std::pair<iterator, bool> find_or_prepare_insert_non_soo(const K& key) {
3806
+ assert(!is_soo());
3087
3807
  prefetch_heap_block();
3088
3808
  auto hash = hash_ref()(key);
3089
3809
  auto seq = probe(common(), hash);
@@ -3094,65 +3814,92 @@ class raw_hash_set {
3094
3814
  if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
3095
3815
  EqualElement<K>{key, eq_ref()},
3096
3816
  PolicyTraits::element(slot_array() + seq.offset(i)))))
3097
- return {seq.offset(i), false};
3817
+ return {iterator_at(seq.offset(i)), false};
3818
+ }
3819
+ auto mask_empty = g.MaskEmpty();
3820
+ if (ABSL_PREDICT_TRUE(mask_empty)) {
3821
+ size_t target = seq.offset(
3822
+ GetInsertionOffset(mask_empty, capacity(), hash, control()));
3823
+ return {iterator_at(PrepareInsertNonSoo(common(), hash,
3824
+ FindInfo{target, seq.index()},
3825
+ GetPolicyFunctions())),
3826
+ true};
3098
3827
  }
3099
- if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break;
3100
3828
  seq.next();
3101
3829
  assert(seq.index() <= capacity() && "full table!");
3102
3830
  }
3103
- return {prepare_insert(hash), true};
3104
3831
  }
3105
3832
 
3106
- // Given the hash of a value not currently in the table, finds the next
3107
- // viable slot index to insert it at.
3108
- //
3109
- // REQUIRES: At least one non-full slot available.
3110
- size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE {
3111
- const bool rehash_for_bug_detection =
3112
- common().should_rehash_for_bug_detection_on_insert();
3113
- if (rehash_for_bug_detection) {
3114
- // Move to a different heap allocation in order to detect bugs.
3115
- const size_t cap = capacity();
3116
- resize(growth_left() > 0 ? cap : NextCapacity(cap));
3117
- }
3118
- auto target = find_first_non_full(common(), hash);
3119
- if (!rehash_for_bug_detection &&
3120
- ABSL_PREDICT_FALSE(growth_left() == 0 &&
3121
- !IsDeleted(control()[target.offset]))) {
3122
- size_t old_capacity = capacity();
3123
- rehash_and_grow_if_necessary();
3124
- // NOTE: It is safe to use `FindFirstNonFullAfterResize`.
3125
- // `FindFirstNonFullAfterResize` must be called right after resize.
3126
- // `rehash_and_grow_if_necessary` may *not* call `resize`
3127
- // and perform `drop_deletes_without_resize` instead. But this
3128
- // could happen only on big tables.
3129
- // For big tables `FindFirstNonFullAfterResize` will always
3130
- // fallback to normal `find_first_non_full`, so it is safe to use it.
3131
- target = HashSetResizeHelper::FindFirstNonFullAfterResize(
3132
- common(), old_capacity, hash);
3133
- }
3134
- common().increment_size();
3135
- set_growth_left(growth_left() - IsEmpty(control()[target.offset]));
3136
- SetCtrl(common(), target.offset, H2(hash), sizeof(slot_type));
3137
- common().maybe_increment_generation_on_insert();
3138
- infoz().RecordInsert(hash, target.probe_length);
3139
- return target.offset;
3833
+ protected:
3834
+ // Asserts that hash and equal functors provided by the user are consistent,
3835
+ // meaning that `eq(k1, k2)` implies `hash(k1)==hash(k2)`.
3836
+ template <class K>
3837
+ void AssertHashEqConsistent(ABSL_ATTRIBUTE_UNUSED const K& key) {
3838
+ #ifndef NDEBUG
3839
+ if (empty()) return;
3840
+
3841
+ const size_t hash_of_arg = hash_ref()(key);
3842
+ const auto assert_consistent = [&](const ctrl_t*, slot_type* slot) {
3843
+ const value_type& element = PolicyTraits::element(slot);
3844
+ const bool is_key_equal =
3845
+ PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element);
3846
+ if (!is_key_equal) return;
3847
+
3848
+ const size_t hash_of_slot =
3849
+ PolicyTraits::apply(HashElement{hash_ref()}, element);
3850
+ const bool is_hash_equal = hash_of_arg == hash_of_slot;
3851
+ if (!is_hash_equal) {
3852
+ // In this case, we're going to crash. Do a couple of other checks for
3853
+ // idempotence issues. Recalculating hash/eq here is also convenient for
3854
+ // debugging with gdb/lldb.
3855
+ const size_t once_more_hash_arg = hash_ref()(key);
3856
+ assert(hash_of_arg == once_more_hash_arg && "hash is not idempotent.");
3857
+ const size_t once_more_hash_slot =
3858
+ PolicyTraits::apply(HashElement{hash_ref()}, element);
3859
+ assert(hash_of_slot == once_more_hash_slot &&
3860
+ "hash is not idempotent.");
3861
+ const bool once_more_eq =
3862
+ PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element);
3863
+ assert(is_key_equal == once_more_eq && "equality is not idempotent.");
3864
+ }
3865
+ assert((!is_key_equal || is_hash_equal) &&
3866
+ "eq(k1, k2) must imply that hash(k1) == hash(k2). "
3867
+ "hash/eq functors are inconsistent.");
3868
+ };
3869
+
3870
+ if (is_soo()) {
3871
+ assert_consistent(/*unused*/ nullptr, soo_slot());
3872
+ return;
3873
+ }
3874
+ // We only do validation for small tables so that it's constant time.
3875
+ if (capacity() > 16) return;
3876
+ IterateOverFullSlots(common(), slot_array(), assert_consistent);
3877
+ #endif
3878
+ }
3879
+
3880
+ // Attempts to find `key` in the table; if it isn't found, returns an iterator
3881
+ // where the value can be inserted into, with the control byte already set to
3882
+ // `key`'s H2. Returns a bool indicating whether an insertion can take place.
3883
+ template <class K>
3884
+ std::pair<iterator, bool> find_or_prepare_insert(const K& key) {
3885
+ AssertHashEqConsistent(key);
3886
+ if (is_soo()) return find_or_prepare_insert_soo(key);
3887
+ return find_or_prepare_insert_non_soo(key);
3140
3888
  }
3141
3889
 
3142
3890
  // Constructs the value in the space pointed by the iterator. This only works
3143
3891
  // after an unsuccessful find_or_prepare_insert() and before any other
3144
3892
  // modifications happen in the raw_hash_set.
3145
3893
  //
3146
- // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where
3147
- // k is the key decomposed from `forward<Args>(args)...`, and the bool
3148
- // returned by find_or_prepare_insert(k) was true.
3894
+ // PRECONDITION: iter was returned from find_or_prepare_insert(k), where k is
3895
+ // the key decomposed from `forward<Args>(args)...`, and the bool returned by
3896
+ // find_or_prepare_insert(k) was true.
3149
3897
  // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...).
3150
3898
  template <class... Args>
3151
- void emplace_at(size_t i, Args&&... args) {
3152
- construct(slot_array() + i, std::forward<Args>(args)...);
3899
+ void emplace_at(iterator iter, Args&&... args) {
3900
+ construct(iter.slot(), std::forward<Args>(args)...);
3153
3901
 
3154
- assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) ==
3155
- iterator_at(i) &&
3902
+ assert(PolicyTraits::apply(FindElement{*this}, *iter) == iter &&
3156
3903
  "constructed value does not match the lookup key");
3157
3904
  }
3158
3905
 
@@ -3160,7 +3907,7 @@ class raw_hash_set {
3160
3907
  return {control() + i, slot_array() + i, common().generation_ptr()};
3161
3908
  }
3162
3909
  const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
3163
- return {control() + i, slot_array() + i, common().generation_ptr()};
3910
+ return const_cast<raw_hash_set*>(this)->iterator_at(i);
3164
3911
  }
3165
3912
 
3166
3913
  reference unchecked_deref(iterator it) { return it.unchecked_deref(); }
@@ -3178,13 +3925,25 @@ class raw_hash_set {
3178
3925
  // side-effect.
3179
3926
  //
3180
3927
  // See `CapacityToGrowth()`.
3181
- size_t growth_left() const { return common().growth_left(); }
3182
- void set_growth_left(size_t gl) { return common().set_growth_left(gl); }
3928
+ size_t growth_left() const {
3929
+ assert(!is_soo());
3930
+ return common().growth_left();
3931
+ }
3932
+
3933
+ GrowthInfo& growth_info() {
3934
+ assert(!is_soo());
3935
+ return common().growth_info();
3936
+ }
3937
+ GrowthInfo growth_info() const {
3938
+ assert(!is_soo());
3939
+ return common().growth_info();
3940
+ }
3183
3941
 
3184
3942
  // Prefetch the heap-allocated memory region to resolve potential TLB and
3185
3943
  // cache misses. This is intended to overlap with execution of calculating the
3186
3944
  // hash for a key.
3187
3945
  void prefetch_heap_block() const {
3946
+ assert(!is_soo());
3188
3947
  #if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
3189
3948
  __builtin_prefetch(control(), 0, 1);
3190
3949
  #endif
@@ -3193,11 +3952,31 @@ class raw_hash_set {
3193
3952
  CommonFields& common() { return settings_.template get<0>(); }
3194
3953
  const CommonFields& common() const { return settings_.template get<0>(); }
3195
3954
 
3196
- ctrl_t* control() const { return common().control(); }
3955
+ ctrl_t* control() const {
3956
+ assert(!is_soo());
3957
+ return common().control();
3958
+ }
3197
3959
  slot_type* slot_array() const {
3960
+ assert(!is_soo());
3198
3961
  return static_cast<slot_type*>(common().slot_array());
3199
3962
  }
3200
- HashtablezInfoHandle infoz() { return common().infoz(); }
3963
+ slot_type* soo_slot() {
3964
+ assert(is_soo());
3965
+ return static_cast<slot_type*>(common().soo_data());
3966
+ }
3967
+ const slot_type* soo_slot() const {
3968
+ return const_cast<raw_hash_set*>(this)->soo_slot();
3969
+ }
3970
+ iterator soo_iterator() {
3971
+ return {SooControl(), soo_slot(), common().generation_ptr()};
3972
+ }
3973
+ const_iterator soo_iterator() const {
3974
+ return const_cast<raw_hash_set*>(this)->soo_iterator();
3975
+ }
3976
+ HashtablezInfoHandle infoz() {
3977
+ assert(!is_soo());
3978
+ return common().infoz();
3979
+ }
3201
3980
 
3202
3981
  hasher& hash_ref() { return settings_.template get<1>(); }
3203
3982
  const hasher& hash_ref() const { return settings_.template get<1>(); }
@@ -3208,12 +3987,9 @@ class raw_hash_set {
3208
3987
  return settings_.template get<3>();
3209
3988
  }
3210
3989
 
3211
- // Make type-specific functions for this type's PolicyFunctions struct.
3212
- static size_t hash_slot_fn(void* set, void* slot) {
3213
- auto* h = static_cast<raw_hash_set*>(set);
3214
- return PolicyTraits::apply(
3215
- HashElement{h->hash_ref()},
3216
- PolicyTraits::element(static_cast<slot_type*>(slot)));
3990
+ static const void* get_hash_ref_fn(const CommonFields& common) {
3991
+ auto* h = reinterpret_cast<const raw_hash_set*>(&common);
3992
+ return &h->hash_ref();
3217
3993
  }
3218
3994
  static void transfer_slot_fn(void* set, void* dst, void* src) {
3219
3995
  auto* h = static_cast<raw_hash_set*>(set);
@@ -3236,13 +4012,18 @@ class raw_hash_set {
3236
4012
  static const PolicyFunctions& GetPolicyFunctions() {
3237
4013
  static constexpr PolicyFunctions value = {
3238
4014
  sizeof(slot_type),
3239
- &raw_hash_set::hash_slot_fn,
4015
+ // TODO(b/328722020): try to type erase
4016
+ // for standard layout and alignof(Hash) <= alignof(CommonFields).
4017
+ std::is_empty<hasher>::value ? &GetHashRefForEmptyHasher
4018
+ : &raw_hash_set::get_hash_ref_fn,
4019
+ PolicyTraits::template get_hash_slot_fn<hasher>(),
3240
4020
  PolicyTraits::transfer_uses_memcpy()
3241
4021
  ? TransferRelocatable<sizeof(slot_type)>
3242
4022
  : &raw_hash_set::transfer_slot_fn,
3243
4023
  (std::is_same<SlotAlloc, std::allocator<slot_type>>::value
3244
4024
  ? &DeallocateStandard<alignof(slot_type)>
3245
4025
  : &raw_hash_set::dealloc_fn),
4026
+ &raw_hash_set::resize_impl,
3246
4027
  };
3247
4028
  return value;
3248
4029
  }
@@ -3252,22 +4033,78 @@ class raw_hash_set {
3252
4033
  // fields that occur after CommonFields.
3253
4034
  absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal,
3254
4035
  allocator_type>
3255
- settings_{CommonFields{}, hasher{}, key_equal{}, allocator_type{}};
4036
+ settings_{CommonFields::CreateDefault<SooEnabled()>(), hasher{},
4037
+ key_equal{}, allocator_type{}};
4038
+ };
4039
+
4040
+ // Friend access for free functions in raw_hash_set.h.
4041
+ struct HashtableFreeFunctionsAccess {
4042
+ template <class Predicate, typename Set>
4043
+ static typename Set::size_type EraseIf(Predicate& pred, Set* c) {
4044
+ if (c->empty()) {
4045
+ return 0;
4046
+ }
4047
+ if (c->is_soo()) {
4048
+ auto it = c->soo_iterator();
4049
+ if (!pred(*it)) {
4050
+ assert(c->size() == 1 && "hash table was modified unexpectedly");
4051
+ return 0;
4052
+ }
4053
+ c->destroy(it.slot());
4054
+ c->common().set_empty_soo();
4055
+ return 1;
4056
+ }
4057
+ ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = c->size();
4058
+ size_t num_deleted = 0;
4059
+ IterateOverFullSlots(
4060
+ c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) {
4061
+ if (pred(Set::PolicyTraits::element(slot))) {
4062
+ c->destroy(slot);
4063
+ EraseMetaOnly(c->common(), static_cast<size_t>(ctrl - c->control()),
4064
+ sizeof(*slot));
4065
+ ++num_deleted;
4066
+ }
4067
+ });
4068
+ // NOTE: IterateOverFullSlots allow removal of the current element, so we
4069
+ // verify the size additionally here.
4070
+ assert(original_size_for_assert - num_deleted == c->size() &&
4071
+ "hash table was modified unexpectedly");
4072
+ return num_deleted;
4073
+ }
4074
+
4075
+ template <class Callback, typename Set>
4076
+ static void ForEach(Callback& cb, Set* c) {
4077
+ if (c->empty()) {
4078
+ return;
4079
+ }
4080
+ if (c->is_soo()) {
4081
+ cb(*c->soo_iterator());
4082
+ return;
4083
+ }
4084
+ using ElementTypeWithConstness = decltype(*c->begin());
4085
+ IterateOverFullSlots(
4086
+ c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) {
4087
+ ElementTypeWithConstness& element = Set::PolicyTraits::element(slot);
4088
+ cb(element);
4089
+ });
4090
+ }
3256
4091
  };
3257
4092
 
3258
4093
  // Erases all elements that satisfy the predicate `pred` from the container `c`.
3259
4094
  template <typename P, typename H, typename E, typename A, typename Predicate>
3260
4095
  typename raw_hash_set<P, H, E, A>::size_type EraseIf(
3261
4096
  Predicate& pred, raw_hash_set<P, H, E, A>* c) {
3262
- const auto initial_size = c->size();
3263
- for (auto it = c->begin(), last = c->end(); it != last;) {
3264
- if (pred(*it)) {
3265
- c->erase(it++);
3266
- } else {
3267
- ++it;
3268
- }
3269
- }
3270
- return initial_size - c->size();
4097
+ return HashtableFreeFunctionsAccess::EraseIf(pred, c);
4098
+ }
4099
+
4100
+ // Calls `cb` for all elements in the container `c`.
4101
+ template <typename P, typename H, typename E, typename A, typename Callback>
4102
+ void ForEach(Callback& cb, raw_hash_set<P, H, E, A>* c) {
4103
+ return HashtableFreeFunctionsAccess::ForEach(cb, c);
4104
+ }
4105
+ template <typename P, typename H, typename E, typename A, typename Callback>
4106
+ void ForEach(Callback& cb, const raw_hash_set<P, H, E, A>* c) {
4107
+ return HashtableFreeFunctionsAccess::ForEach(cb, c);
3271
4108
  }
3272
4109
 
3273
4110
  namespace hashtable_debug_internal {
@@ -3278,6 +4115,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> {
3278
4115
 
3279
4116
  static size_t GetNumProbes(const Set& set,
3280
4117
  const typename Set::key_type& key) {
4118
+ if (set.is_soo()) return 0;
3281
4119
  size_t num_probes = 0;
3282
4120
  size_t hash = set.hash_ref()(key);
3283
4121
  auto seq = probe(set.common(), hash);
@@ -3301,7 +4139,8 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> {
3301
4139
  static size_t AllocatedByteSize(const Set& c) {
3302
4140
  size_t capacity = c.capacity();
3303
4141
  if (capacity == 0) return 0;
3304
- size_t m = c.common().alloc_size(sizeof(Slot), alignof(Slot));
4142
+ size_t m =
4143
+ c.is_soo() ? 0 : c.common().alloc_size(sizeof(Slot), alignof(Slot));
3305
4144
 
3306
4145
  size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr));
3307
4146
  if (per_slot != ~size_t{}) {
@@ -3321,5 +4160,7 @@ ABSL_NAMESPACE_END
3321
4160
  } // namespace absl
3322
4161
 
3323
4162
  #undef ABSL_SWISSTABLE_ENABLE_GENERATIONS
4163
+ #undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED
4164
+ #undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN
3324
4165
 
3325
4166
  #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_