grpc 1.67.0 → 1.69.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Makefile +58 -46
- data/include/grpc/compression.h +1 -2
- data/include/grpc/credentials.h +1 -2
- data/include/grpc/event_engine/endpoint_config.h +2 -2
- data/include/grpc/event_engine/event_engine.h +12 -9
- data/include/grpc/event_engine/extensible.h +2 -2
- data/include/grpc/event_engine/internal/memory_allocator_impl.h +4 -4
- data/include/grpc/event_engine/memory_allocator.h +3 -4
- data/include/grpc/event_engine/memory_request.h +1 -2
- data/include/grpc/event_engine/slice.h +3 -4
- data/include/grpc/event_engine/slice_buffer.h +6 -7
- data/include/grpc/grpc.h +1 -2
- data/include/grpc/grpc_audit_logging.h +3 -3
- data/include/grpc/grpc_crl_provider.h +4 -4
- data/include/grpc/grpc_posix.h +1 -2
- data/include/grpc/grpc_security.h +1 -2
- data/include/grpc/impl/call.h +2 -2
- data/include/grpc/impl/channel_arg_names.h +2 -0
- data/include/grpc/impl/grpc_types.h +1 -2
- data/include/grpc/impl/slice_type.h +1 -2
- data/include/grpc/passive_listener.h +2 -2
- data/include/grpc/support/alloc.h +1 -2
- data/include/grpc/support/json.h +1 -2
- data/include/grpc/support/log.h +1 -2
- data/include/grpc/support/metrics.h +7 -3
- data/include/grpc/support/port_platform.h +6 -1
- data/include/grpc/support/sync_posix.h +1 -2
- data/include/grpc/support/time.h +1 -2
- data/src/core/channelz/channel_trace.cc +5 -6
- data/src/core/channelz/channel_trace.h +5 -7
- data/src/core/channelz/channelz.cc +5 -6
- data/src/core/channelz/channelz.h +8 -10
- data/src/core/channelz/channelz_registry.cc +6 -7
- data/src/core/channelz/channelz_registry.h +4 -5
- data/src/core/client_channel/backup_poller.cc +21 -11
- data/src/core/client_channel/client_channel.cc +24 -20
- data/src/core/client_channel/client_channel.h +4 -2
- data/src/core/client_channel/client_channel_factory.cc +2 -2
- data/src/core/client_channel/client_channel_factory.h +1 -2
- data/src/core/client_channel/client_channel_filter.cc +27 -24
- data/src/core/client_channel/client_channel_filter.h +11 -11
- data/src/core/client_channel/client_channel_internal.h +2 -3
- data/src/core/client_channel/client_channel_plugin.cc +2 -4
- data/src/core/client_channel/client_channel_service_config.cc +2 -3
- data/src/core/client_channel/client_channel_service_config.h +4 -6
- data/src/core/client_channel/config_selector.h +4 -7
- data/src/core/client_channel/connector.h +3 -3
- data/src/core/client_channel/direct_channel.cc +3 -3
- data/src/core/client_channel/dynamic_filters.cc +10 -8
- data/src/core/client_channel/dynamic_filters.h +8 -7
- data/src/core/client_channel/global_subchannel_pool.cc +2 -2
- data/src/core/client_channel/global_subchannel_pool.h +2 -3
- data/src/core/client_channel/lb_metadata.h +2 -3
- data/src/core/client_channel/load_balanced_call_destination.cc +2 -3
- data/src/core/client_channel/load_balanced_call_destination.h +0 -1
- data/src/core/client_channel/local_subchannel_pool.cc +2 -3
- data/src/core/client_channel/local_subchannel_pool.h +1 -1
- data/src/core/client_channel/retry_filter.cc +5 -7
- data/src/core/client_channel/retry_filter.h +4 -7
- data/src/core/client_channel/retry_filter_legacy_call_data.cc +12 -12
- data/src/core/client_channel/retry_filter_legacy_call_data.h +11 -12
- data/src/core/client_channel/retry_service_config.cc +6 -8
- data/src/core/client_channel/retry_service_config.h +3 -5
- data/src/core/client_channel/retry_throttle.cc +3 -4
- data/src/core/client_channel/retry_throttle.h +4 -7
- data/src/core/client_channel/subchannel.cc +12 -15
- data/src/core/client_channel/subchannel.h +17 -16
- data/src/core/client_channel/subchannel_interface_internal.h +1 -1
- data/src/core/client_channel/subchannel_pool_interface.cc +1 -3
- data/src/core/client_channel/subchannel_pool_interface.h +2 -3
- data/src/core/client_channel/subchannel_stream_client.cc +6 -9
- data/src/core/client_channel/subchannel_stream_client.h +8 -10
- data/src/core/{lib/config → config}/config_vars.cc +4 -5
- data/src/core/{lib/config → config}/config_vars.h +4 -5
- data/src/core/{lib/config → config}/config_vars_non_generated.cc +3 -3
- data/src/core/{lib/config → config}/core_configuration.cc +4 -4
- data/src/core/{lib/config → config}/core_configuration.h +5 -6
- data/src/core/{lib/config → config}/load_config.cc +3 -5
- data/src/core/{lib/config → config}/load_config.h +4 -5
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.cc +9 -9
- data/src/core/ext/filters/backend_metrics/backend_metric_filter.h +0 -1
- data/src/core/ext/filters/census/grpc_context.cc +1 -2
- data/src/core/ext/filters/channel_idle/idle_filter_state.cc +1 -2
- data/src/core/ext/filters/channel_idle/idle_filter_state.h +0 -1
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.cc +16 -18
- data/src/core/ext/filters/channel_idle/legacy_channel_idle_filter.h +6 -8
- data/src/core/ext/filters/fault_injection/fault_injection_filter.cc +10 -7
- data/src/core/ext/filters/fault_injection/fault_injection_filter.h +1 -3
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.cc +2 -3
- data/src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h +4 -7
- data/src/core/ext/filters/gcp_authentication/gcp_authentication_filter.cc +55 -17
- data/src/core/ext/filters/gcp_authentication/gcp_authentication_filter.h +28 -13
- data/src/core/ext/filters/gcp_authentication/gcp_authentication_service_config_parser.cc +2 -3
- data/src/core/ext/filters/gcp_authentication/gcp_authentication_service_config_parser.h +2 -3
- data/src/core/ext/filters/http/client/http_client_filter.cc +12 -7
- data/src/core/ext/filters/http/client/http_client_filter.h +0 -1
- data/src/core/ext/filters/http/client_authority_filter.cc +7 -6
- data/src/core/ext/filters/http/client_authority_filter.h +0 -1
- data/src/core/ext/filters/http/http_filters_plugin.cc +1 -2
- data/src/core/ext/filters/http/message_compress/compression_filter.cc +24 -10
- data/src/core/ext/filters/http/message_compress/compression_filter.h +1 -4
- data/src/core/ext/filters/http/server/http_server_filter.cc +11 -6
- data/src/core/ext/filters/http/server/http_server_filter.h +0 -1
- data/src/core/ext/filters/message_size/message_size_filter.cc +13 -7
- data/src/core/ext/filters/message_size/message_size_filter.h +2 -4
- data/src/core/ext/filters/rbac/rbac_filter.cc +6 -6
- data/src/core/ext/filters/rbac/rbac_filter.h +0 -2
- data/src/core/ext/filters/rbac/rbac_service_config_parser.cc +14 -8
- data/src/core/ext/filters/rbac/rbac_service_config_parser.h +2 -4
- data/src/core/ext/filters/stateful_session/stateful_session_filter.cc +11 -6
- data/src/core/ext/filters/stateful_session/stateful_session_filter.h +2 -4
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.cc +2 -3
- data/src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h +3 -5
- data/src/core/ext/transport/chttp2/alpn/alpn.cc +1 -2
- data/src/core/ext/transport/chttp2/alpn/alpn.h +1 -2
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +20 -24
- data/src/core/ext/transport/chttp2/client/chttp2_connector.h +4 -5
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +151 -90
- data/src/core/ext/transport/chttp2/server/chttp2_server.h +2 -2
- data/src/core/ext/transport/chttp2/transport/bin_decoder.cc +6 -6
- data/src/core/ext/transport/chttp2/transport/bin_decoder.h +2 -3
- data/src/core/ext/transport/chttp2/transport/bin_encoder.cc +4 -6
- data/src/core/ext/transport/chttp2/transport/bin_encoder.h +1 -2
- data/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc +53 -0
- data/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.h +72 -0
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +152 -95
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +5 -6
- data/src/core/ext/transport/chttp2/transport/context_list_entry.h +1 -2
- data/src/core/ext/transport/chttp2/transport/decode_huff.h +4 -4
- data/src/core/ext/transport/chttp2/transport/flow_control.cc +3 -5
- data/src/core/ext/transport/chttp2/transport/flow_control.h +2 -4
- data/src/core/ext/transport/chttp2/transport/frame.cc +22 -4
- data/src/core/ext/transport/chttp2/transport/frame.h +13 -4
- data/src/core/ext/transport/chttp2/transport/frame_data.cc +17 -13
- data/src/core/ext/transport/chttp2/transport/frame_data.h +2 -4
- data/src/core/ext/transport/chttp2/transport/frame_goaway.cc +3 -5
- data/src/core/ext/transport/chttp2/transport/frame_goaway.h +1 -2
- data/src/core/ext/transport/chttp2/transport/frame_ping.cc +2 -4
- data/src/core/ext/transport/chttp2/transport/frame_ping.h +1 -2
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc +4 -5
- data/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +1 -2
- data/src/core/ext/transport/chttp2/transport/frame_security.cc +82 -0
- data/src/core/ext/transport/chttp2/transport/frame_security.h +44 -0
- data/src/core/ext/transport/chttp2/transport/frame_settings.cc +19 -5
- data/src/core/ext/transport/chttp2/transport/frame_settings.h +3 -4
- data/src/core/ext/transport/chttp2/transport/frame_window_update.cc +21 -3
- data/src/core/ext/transport/chttp2/transport/frame_window_update.h +1 -2
- data/src/core/ext/transport/chttp2/transport/hpack_constants.h +2 -2
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +10 -11
- data/src/core/ext/transport/chttp2/transport/hpack_encoder.h +6 -8
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.cc +2 -2
- data/src/core/ext/transport/chttp2/transport/hpack_encoder_table.h +4 -5
- data/src/core/ext/transport/chttp2/transport/hpack_parse_result.cc +2 -4
- data/src/core/ext/transport/chttp2/transport/hpack_parse_result.h +4 -6
- data/src/core/ext/transport/chttp2/transport/hpack_parser.cc +4 -6
- data/src/core/ext/transport/chttp2/transport/hpack_parser.h +3 -5
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.cc +1 -3
- data/src/core/ext/transport/chttp2/transport/hpack_parser_table.h +2 -4
- data/src/core/ext/transport/chttp2/transport/http2_settings.cc +10 -2
- data/src/core/ext/transport/chttp2/transport/http2_settings.h +10 -4
- data/src/core/ext/transport/chttp2/transport/internal.h +37 -99
- data/src/core/ext/transport/chttp2/transport/legacy_frame.h +1 -0
- data/src/core/ext/transport/chttp2/transport/parsing.cc +44 -9
- data/src/core/ext/transport/chttp2/transport/ping_abuse_policy.cc +3 -3
- data/src/core/ext/transport/chttp2/transport/ping_abuse_policy.h +3 -3
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.cc +2 -2
- data/src/core/ext/transport/chttp2/transport/ping_callbacks.h +3 -5
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc +4 -5
- data/src/core/ext/transport/chttp2/transport/ping_rate_policy.h +2 -4
- data/src/core/ext/transport/chttp2/transport/stream_lists.cc +4 -3
- data/src/core/ext/transport/chttp2/transport/stream_lists.h +65 -0
- data/src/core/ext/transport/chttp2/transport/varint.cc +2 -2
- data/src/core/ext/transport/chttp2/transport/varint.h +1 -2
- data/src/core/ext/transport/chttp2/transport/write_size_policy.cc +2 -2
- data/src/core/ext/transport/chttp2/transport/write_size_policy.h +3 -4
- data/src/core/ext/transport/chttp2/transport/writing.cc +11 -11
- data/src/core/ext/transport/inproc/inproc_transport.cc +6 -7
- data/src/core/ext/transport/inproc/legacy_inproc_transport.cc +12 -14
- data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb.h +35 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.c +25 -18
- data/src/core/ext/upb-gen/envoy/admin/v3/certs.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb.h +36 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.c +24 -18
- data/src/core/ext/upb-gen/envoy/admin/v3/clusters.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb.h +39 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.c +29 -22
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb.h +137 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.c +105 -84
- data/src/core/ext/upb-gen/envoy/admin/v3/config_dump_shared.upb_minitable.h +24 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb.h +10 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/init_dump.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/envoy/admin/v3/listeners.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/admin/v3/memory.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/admin/v3/metrics.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/admin/v3/mutex_stats.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb.h +19 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.c +16 -12
- data/src/core/ext/upb-gen/envoy/admin/v3/server_info.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/admin/v3/tap.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.c +7 -1
- data/src/core/ext/upb-gen/envoy/annotations/deprecation.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/envoy/annotations/resource.upb.h +7 -6
- data/src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/envoy/annotations/resource.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb.h +61 -4
- data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.c +66 -48
- data/src/core/ext/upb-gen/envoy/config/accesslog/v3/accesslog.upb_minitable.h +21 -4
- data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb.h +183 -4
- data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.c +131 -106
- data/src/core/ext/upb-gen/envoy/config/bootstrap/v3/bootstrap.upb_minitable.h +28 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb.h +31 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.c +22 -17
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/circuit_breaker.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb.h +185 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.c +149 -122
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/cluster.upb_minitable.h +30 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/filter.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb.h +56 -4
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.c +30 -27
- data/src/core/ext/upb-gen/envoy/config/cluster/v3/outlier_detection.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb.h +96 -4
- data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.c +80 -63
- data/src/core/ext/upb-gen/envoy/config/common/matcher/v3/matcher.upb_minitable.h +20 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb.h +44 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.c +36 -26
- data/src/core/ext/upb-gen/envoy/config/core/v3/address.upb_minitable.h +13 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/config/core/v3/backoff.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb.h +109 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.c +108 -77
- data/src/core/ext/upb-gen/envoy/config/core/v3/base.upb_minitable.h +34 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb.h +46 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.c +36 -27
- data/src/core/ext/upb-gen/envoy/config/core/v3/config_source.upb_minitable.h +12 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/event_service_config.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/extension.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb.h +10 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_method_list.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb.h +71 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.c +61 -45
- data/src/core/ext/upb-gen/envoy/config/core/v3/grpc_service.upb_minitable.h +19 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb.h +92 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.c +55 -44
- data/src/core/ext/upb-gen/envoy/config/core/v3/health_check.upb_minitable.h +14 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_service.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/http_uri.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb.h +99 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.c +85 -67
- data/src/core/ext/upb-gen/envoy/config/core/v3/protocol.upb_minitable.h +21 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/proxy_protocol.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/envoy/config/core/v3/resolver.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb.h +10 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/socket_option.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb.h +16 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/envoy/config/core/v3/substitution_format_string.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/config/core/v3/udp_socket_config.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb.h +35 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.c +23 -17
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb.h +48 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.c +41 -32
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/endpoint_components.upb_minitable.h +12 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb.h +44 -4
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.c +27 -19
- data/src/core/ext/upb-gen/envoy/config/endpoint/v3/load_report.upb_minitable.h +11 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/api_listener.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb.h +83 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.c +53 -41
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener.upb_minitable.h +15 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb.h +66 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.c +47 -38
- data/src/core/ext/upb-gen/envoy/config/listener/v3/listener_components.upb_minitable.h +12 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb.h +27 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.c +17 -14
- data/src/core/ext/upb-gen/envoy/config/listener/v3/quic_config.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb.h +11 -4
- data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/envoy/config/listener/v3/udp_listener_config.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/metrics_service.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb.h +33 -4
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.c +34 -24
- data/src/core/ext/upb-gen/envoy/config/metrics/v3/stats.upb_minitable.h +13 -4
- data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb.h +51 -4
- data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.c +41 -29
- data/src/core/ext/upb-gen/envoy/config/overload/v3/overload.upb_minitable.h +15 -4
- data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb.h +96 -4
- data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.c +68 -55
- data/src/core/ext/upb-gen/envoy/config/rbac/v3/rbac.upb_minitable.h +16 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb.h +52 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.c +24 -19
- data/src/core/ext/upb-gen/envoy/config/route/v3/route.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb.h +435 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.c +306 -246
- data/src/core/ext/upb-gen/envoy/config/route/v3/route_components.upb_minitable.h +63 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb.h +14 -4
- data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.c +13 -8
- data/src/core/ext/upb-gen/envoy/config/route/v3/scoped_route.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb.h +69 -4
- data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.c +59 -45
- data/src/core/ext/upb-gen/envoy/config/tap/v3/common.upb_minitable.h +17 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/envoy/config/trace/v3/datadog.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/dynamic_ot.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/envoy/config/trace/v3/http_tracer.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/lightstep.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb.h +11 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.c +9 -6
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opencensus.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb.h +16 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.c +10 -7
- data/src/core/ext/upb-gen/envoy/config/trace/v3/opentelemetry.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/service.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb.h +11 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/envoy/config/trace/v3/skywalking.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.c +3 -1
- data/src/core/ext/upb-gen/envoy/config/trace/v3/trace.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb.h +13 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.c +13 -9
- data/src/core/ext/upb-gen/envoy/config/trace/v3/xray.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/config/trace/v3/zipkin.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb.h +102 -4
- data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.c +72 -54
- data/src/core/ext/upb-gen/envoy/data/accesslog/v3/accesslog.upb_minitable.h +21 -4
- data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/extensions/clusters/aggregate/v3/cluster.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.c +18 -11
- data/src/core/ext/upb-gen/envoy/extensions/filters/common/fault/v3/fault.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb.h +24 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.c +18 -13
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/fault/v3/fault.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.c +17 -11
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb.h +15 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.c +14 -10
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/rbac/v3/rbac.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb.h +21 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.c +14 -10
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/router/v3/router.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h +181 -4
- data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.c +132 -109
- data/src/core/ext/upb-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb_minitable.h +26 -4
- data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.c +12 -9
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb.h +21 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.c +24 -17
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/common/v3/common.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/pick_first/v3/pick_first.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h +15 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.c +11 -8
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb.h +106 -0
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.c +52 -0
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upb_minitable.h +32 -0
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.c +3 -1
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/cert.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb.h +59 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.c +42 -32
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/common.upb_minitable.h +13 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.c +18 -13
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/secret.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb.h +77 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.c +52 -43
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls.upb_minitable.h +12 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb.h +42 -4
- data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.c +32 -26
- data/src/core/ext/upb-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/ads.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb.h +86 -4
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.c +63 -47
- data/src/core/ext/upb-gen/envoy/service/discovery/v3/discovery.upb_minitable.h +19 -4
- data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb.h +14 -4
- data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/envoy/service/load_stats/v3/lrs.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb.h +45 -4
- data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.c +32 -25
- data/src/core/ext/upb-gen/envoy/service/status/v3/csds.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/type/http/v3/cookie.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb.h +14 -4
- data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.c +14 -8
- data/src/core/ext/upb-gen/envoy/type/http/v3/path_transformation.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/filter_state.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.c +8 -1
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/http_inputs.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/metadata.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/node.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/number.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/path.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb.h +11 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.c +15 -10
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/regex.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.c +5 -1
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/status_code_input.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb.h +14 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/string.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb.h +12 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/struct.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb.h +22 -4
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.c +20 -14
- data/src/core/ext/upb-gen/envoy/type/matcher/v3/value.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb.h +18 -4
- data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.c +19 -10
- data/src/core/ext/upb-gen/envoy/type/metadata/v3/metadata.upb_minitable.h +12 -4
- data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb.h +17 -4
- data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.c +18 -11
- data/src/core/ext/upb-gen/envoy/type/tracing/v3/custom_tag.upb_minitable.h +10 -4
- data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.c +10 -5
- data/src/core/ext/upb-gen/envoy/type/v3/hash_policy.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/type/v3/http.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.c +3 -1
- data/src/core/ext/upb-gen/envoy/type/v3/http.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/type/v3/http_status.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/v3/percent.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.c +5 -1
- data/src/core/ext/upb-gen/envoy/type/v3/percent.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/v3/range.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.c +6 -1
- data/src/core/ext/upb-gen/envoy/type/v3/range.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_strategy.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.c +3 -1
- data/src/core/ext/upb-gen/envoy/type/v3/ratelimit_unit.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb.h +5 -4
- data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/envoy/type/v3/semantic_version.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb.h +9 -4
- data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/envoy/type/v3/token_bucket.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/api/annotations.upb.h +7 -6
- data/src/core/ext/upb-gen/google/api/annotations.upb_minitable.c +5 -2
- data/src/core/ext/upb-gen/google/api/annotations.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb.h +87 -4
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.c +69 -54
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/checked.upb_minitable.h +18 -4
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb.h +85 -4
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.c +71 -53
- data/src/core/ext/upb-gen/google/api/expr/v1alpha1/syntax.upb_minitable.h +21 -4
- data/src/core/ext/upb-gen/google/api/http.upb.h +18 -4
- data/src/core/ext/upb-gen/google/api/http.upb_minitable.c +13 -8
- data/src/core/ext/upb-gen/google/api/http.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/google/api/httpbody.upb.h +10 -4
- data/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/any.upb.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/descriptor.upb.h +202 -4
- data/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c +241 -206
- data/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.h +55 -21
- data/src/core/ext/upb-gen/google/protobuf/duration.upb.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/empty.upb.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/struct.upb.h +27 -4
- data/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c +20 -14
- data/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/google/protobuf/timestamp.upb.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/wrappers.upb.h +6 -4
- data/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c +12 -1
- data/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.h +14 -4
- data/src/core/ext/upb-gen/google/rpc/status.upb.h +10 -4
- data/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/google/rpc/status.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb.h +12 -4
- data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.c +12 -6
- data/src/core/ext/upb-gen/opencensus/proto/trace/v1/trace_config.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb.h +13 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb.h +63 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c +51 -37
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.h +17 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb.h +10 -4
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c +9 -5
- data/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb.h +6 -4
- data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c +5 -1
- data/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb.h +29 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c +31 -20
- data/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.h +14 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb.h +12 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c +9 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb.h +57 -4
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c +33 -22
- data/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.h +14 -4
- data/src/core/ext/upb-gen/udpa/annotations/migrate.upb.h +15 -14
- data/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c +22 -12
- data/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/udpa/annotations/security.upb.h +7 -6
- data/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb.h +5 -4
- data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/udpa/annotations/status.upb.h +7 -6
- data/src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/udpa/annotations/status.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/udpa/annotations/versioning.upb.h +7 -6
- data/src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/udpa/annotations/versioning.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/validate/validate.upb.h +89 -6
- data/src/core/ext/upb-gen/validate/validate.upb_minitable.c +116 -87
- data/src/core/ext/upb-gen/validate/validate.upb_minitable.h +29 -5
- data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb.h +15 -14
- data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.c +22 -12
- data/src/core/ext/upb-gen/xds/annotations/v3/migrate.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/xds/annotations/v3/security.upb.h +7 -6
- data/src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/xds/annotations/v3/security.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb.h +5 -4
- data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/xds/annotations/v3/sensitive.upb_minitable.h +5 -4
- data/src/core/ext/upb-gen/xds/annotations/v3/status.upb.h +11 -10
- data/src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.c +20 -11
- data/src/core/ext/upb-gen/xds/annotations/v3/status.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb.h +7 -6
- data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.c +10 -6
- data/src/core/ext/upb-gen/xds/annotations/v3/versioning.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/core/v3/authority.upb.h +5 -4
- data/src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/xds/core/v3/authority.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/core/v3/cidr.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/cidr.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb.h +11 -4
- data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/xds/core/v3/collection_entry.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/context_params.upb.h +11 -4
- data/src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/xds/core/v3/context_params.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/extension.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/extension.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource.upb.h +9 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.c +8 -5
- data/src/core/ext/upb-gen/xds/core/v3/resource.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb.h +14 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/xds/core/v3/resource_locator.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/core/v3/resource_name.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb.h +23 -4
- data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c +12 -6
- data/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.h +9 -4
- data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/cel.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb.h +12 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/xds/type/matcher/v3/domain.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb.h +5 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.c +4 -1
- data/src/core/ext/upb-gen/xds/type/matcher/v3/http_inputs.upb_minitable.h +6 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb.h +17 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.c +12 -8
- data/src/core/ext/upb-gen/xds/type/matcher/v3/ip.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb.h +63 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.c +55 -43
- data/src/core/ext/upb-gen/xds/type/matcher/v3/matcher.upb_minitable.h +15 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb.h +41 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.c +30 -22
- data/src/core/ext/upb-gen/xds/type/matcher/v3/range.upb_minitable.h +11 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.c +8 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/regex.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb.h +12 -4
- data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.c +11 -7
- data/src/core/ext/upb-gen/xds/type/matcher/v3/string.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/type/v3/cel.upb.h +13 -4
- data/src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.c +13 -9
- data/src/core/ext/upb-gen/xds/type/v3/cel.upb_minitable.h +7 -4
- data/src/core/ext/upb-gen/xds/type/v3/range.upb.h +6 -4
- data/src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.c +6 -1
- data/src/core/ext/upb-gen/xds/type/v3/range.upb_minitable.h +8 -4
- data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb.h +7 -4
- data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.c +7 -4
- data/src/core/ext/upb-gen/xds/type/v3/typed_struct.upb_minitable.h +6 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/certs.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/clusters.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/config_dump_shared.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/init_dump.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/listeners.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/memory.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/metrics.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/mutex_stats.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/server_info.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/admin/v3/tap.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/annotations/deprecation.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/annotations/resource.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/accesslog/v3/accesslog.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/bootstrap/v3/bootstrap.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/circuit_breaker.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/cluster.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/filter.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/cluster/v3/outlier_detection.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/common/matcher/v3/matcher.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/address.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/backoff.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/base.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/config_source.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/event_service_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/extension.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_method_list.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/grpc_service.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/health_check.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_service.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/http_uri.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/protocol.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/proxy_protocol.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/resolver.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/socket_option.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/substitution_format_string.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/core/v3/udp_socket_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/endpoint_components.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/endpoint/v3/load_report.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/api_listener.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/listener_components.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/quic_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/listener/v3/udp_listener_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/metrics_service.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/metrics/v3/stats.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/overload/v3/overload.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/rbac/v3/rbac.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/route_components.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/route/v3/scoped_route.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/tap/v3/common.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/datadog.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/dynamic_ot.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/http_tracer.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/lightstep.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opencensus.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/opentelemetry.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/service.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/skywalking.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/trace.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/xray.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/config/trace/v3/zipkin.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/router/v3/router.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.c +53 -0
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/http_11_proxy/v3/upstream_http_11_connect.upbdefs.h +33 -0
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/cert.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/common.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/secret.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/extensions/upstreams/http/v3/http_protocol_options.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/ads.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/service/discovery/v3/discovery.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/service/load_stats/v3/lrs.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/service/status/v3/csds.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/http/v3/cookie.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/http/v3/path_transformation.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/filter_state.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/http_inputs.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/metadata.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/node.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/number.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/path.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/regex.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/status_code_input.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/string.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/struct.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/matcher/v3/value.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/metadata/v3/metadata.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/tracing/v3/custom_tag.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/hash_policy.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/http.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/http_status.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/percent.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/range.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_strategy.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/ratelimit_unit.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/semantic_version.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/envoy/type/v3/token_bucket.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/api/annotations.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/checked.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/api/expr/v1alpha1/syntax.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/api/http.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/api/http.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/api/httpbody.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/any.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.c +486 -484
- data/src/core/ext/upbdefs-gen/google/protobuf/descriptor.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/duration.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/empty.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/struct.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/opencensus/proto/trace/v1/trace_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/udpa/annotations/status.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/udpa/annotations/versioning.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/validate/validate.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/validate/validate.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/migrate.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/security.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/sensitive.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/status.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/annotations/v3/versioning.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/authority.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/cidr.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/collection_entry.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/context_params.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/extension.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource_locator.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/core/v3/resource_name.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/cel.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/domain.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/http_inputs.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/ip.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/matcher.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/range.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/regex.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/matcher/v3/string.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/v3/cel.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/v3/range.upbdefs.h +5 -4
- data/src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.c +2 -1
- data/src/core/ext/upbdefs-gen/xds/type/v3/typed_struct.upbdefs.h +5 -4
- data/src/core/filter/blackboard.cc +33 -0
- data/src/core/filter/blackboard.h +70 -0
- data/src/core/handshaker/endpoint_info/endpoint_info_handshaker.cc +4 -5
- data/src/core/handshaker/endpoint_info/endpoint_info_handshaker.h +1 -1
- data/src/core/handshaker/handshaker.cc +8 -11
- data/src/core/handshaker/handshaker.h +8 -10
- data/src/core/handshaker/handshaker_registry.cc +1 -2
- data/src/core/handshaker/handshaker_registry.h +2 -2
- data/src/core/handshaker/http_connect/http_connect_handshaker.cc +14 -11
- data/src/core/handshaker/http_connect/http_connect_handshaker.h +1 -1
- data/src/core/handshaker/http_connect/http_proxy_mapper.cc +7 -9
- data/src/core/handshaker/http_connect/http_proxy_mapper.h +3 -4
- data/src/core/handshaker/http_connect/xds_http_proxy_mapper.cc +57 -0
- data/src/core/handshaker/http_connect/xds_http_proxy_mapper.h +46 -0
- data/src/core/handshaker/proxy_mapper.h +2 -3
- data/src/core/handshaker/proxy_mapper_registry.cc +2 -2
- data/src/core/handshaker/proxy_mapper_registry.h +2 -3
- data/src/core/handshaker/security/secure_endpoint.cc +12 -14
- data/src/core/handshaker/security/secure_endpoint.h +2 -3
- data/src/core/handshaker/security/security_handshaker.cc +15 -21
- data/src/core/handshaker/security/security_handshaker.h +3 -4
- data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.cc +10 -11
- data/src/core/handshaker/tcp_connect/tcp_connect_handshaker.h +1 -1
- data/src/core/lib/address_utils/parse_address.cc +3 -4
- data/src/core/lib/address_utils/parse_address.h +2 -4
- data/src/core/lib/address_utils/sockaddr_utils.cc +4 -6
- data/src/core/lib/address_utils/sockaddr_utils.h +1 -3
- data/src/core/lib/channel/call_finalization.h +2 -2
- data/src/core/lib/channel/channel_args.cc +4 -6
- data/src/core/lib/channel/channel_args.h +11 -13
- data/src/core/lib/channel/channel_args_preconditioning.cc +2 -2
- data/src/core/lib/channel/channel_args_preconditioning.h +4 -4
- data/src/core/lib/channel/channel_stack.cc +5 -4
- data/src/core/lib/channel/channel_stack.h +17 -17
- data/src/core/lib/channel/channel_stack_builder.cc +2 -2
- data/src/core/lib/channel/channel_stack_builder.h +3 -4
- data/src/core/lib/channel/channel_stack_builder_impl.cc +5 -7
- data/src/core/lib/channel/channel_stack_builder_impl.h +15 -3
- data/src/core/lib/channel/connected_channel.cc +9 -11
- data/src/core/lib/channel/promise_based_filter.cc +33 -24
- data/src/core/lib/channel/promise_based_filter.h +45 -16
- data/src/core/lib/channel/status_util.cc +1 -3
- data/src/core/lib/channel/status_util.h +3 -3
- data/src/core/lib/compression/compression.cc +3 -5
- data/src/core/lib/compression/compression_internal.cc +5 -7
- data/src/core/lib/compression/compression_internal.h +3 -5
- data/src/core/lib/compression/message_compress.cc +3 -6
- data/src/core/lib/debug/trace.cc +5 -6
- data/src/core/lib/debug/trace_flags.cc +1 -2
- data/src/core/lib/debug/trace_impl.h +2 -2
- data/src/core/lib/event_engine/ares_resolver.cc +11 -15
- data/src/core/lib/event_engine/ares_resolver.h +7 -10
- data/src/core/lib/event_engine/cf_engine/cf_engine.cc +2 -4
- data/src/core/lib/event_engine/cf_engine/cf_engine.h +1 -1
- data/src/core/lib/event_engine/cf_engine/cfstream_endpoint.cc +1 -2
- data/src/core/lib/event_engine/cf_engine/cfstream_endpoint.h +4 -6
- data/src/core/lib/event_engine/cf_engine/dns_service_resolver.cc +9 -7
- data/src/core/lib/event_engine/cf_engine/dns_service_resolver.h +3 -5
- data/src/core/lib/event_engine/channel_args_endpoint_config.cc +3 -4
- data/src/core/lib/event_engine/channel_args_endpoint_config.h +2 -3
- data/src/core/lib/event_engine/common_closures.h +3 -3
- data/src/core/lib/event_engine/default_event_engine.cc +7 -8
- data/src/core/lib/event_engine/default_event_engine.h +4 -4
- data/src/core/lib/event_engine/default_event_engine_factory.cc +2 -2
- data/src/core/lib/event_engine/default_event_engine_factory.h +2 -2
- data/src/core/lib/event_engine/event_engine.cc +2 -2
- data/src/core/lib/event_engine/extensions/can_track_errors.h +2 -2
- data/src/core/lib/event_engine/extensions/chaotic_good_extension.h +1 -2
- data/src/core/lib/event_engine/extensions/supports_fd.h +20 -3
- data/src/core/lib/event_engine/extensions/tcp_trace.h +0 -1
- data/src/core/lib/event_engine/forkable.cc +3 -3
- data/src/core/lib/event_engine/forkable.h +2 -2
- data/src/core/lib/event_engine/grpc_polled_fd.h +3 -4
- data/src/core/lib/event_engine/handle_containers.h +2 -3
- data/src/core/lib/event_engine/memory_allocator_factory.h +3 -4
- data/src/core/lib/event_engine/nameser.h +1 -1
- data/src/core/lib/event_engine/poller.h +2 -2
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.cc +10 -12
- data/src/core/lib/event_engine/posix_engine/ev_epoll1_linux.h +4 -5
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.cc +17 -20
- data/src/core/lib/event_engine/posix_engine/ev_poll_posix.h +4 -5
- data/src/core/lib/event_engine/posix_engine/event_poller.h +3 -4
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.cc +4 -5
- data/src/core/lib/event_engine/posix_engine/event_poller_posix_default.h +2 -2
- data/src/core/lib/event_engine/posix_engine/grpc_polled_fd_posix.h +4 -6
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.cc +2 -3
- data/src/core/lib/event_engine/posix_engine/internal_errqueue.h +1 -2
- data/src/core/lib/event_engine/posix_engine/lockfree_event.cc +5 -6
- data/src/core/lib/event_engine/posix_engine/lockfree_event.h +2 -3
- data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.cc +2 -3
- data/src/core/lib/event_engine/posix_engine/native_posix_dns_resolver.h +2 -3
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.cc +15 -17
- data/src/core/lib/event_engine/posix_engine/posix_endpoint.h +8 -9
- data/src/core/lib/event_engine/posix_engine/posix_engine.cc +44 -28
- data/src/core/lib/event_engine/posix_engine/posix_engine.h +16 -14
- data/src/core/lib/event_engine/posix_engine/posix_engine_closure.h +3 -3
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.cc +7 -9
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener.h +12 -11
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.cc +4 -6
- data/src/core/lib/event_engine/posix_engine/posix_engine_listener_utils.h +1 -2
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.cc +8 -11
- data/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +7 -8
- data/src/core/lib/event_engine/posix_engine/timer.cc +4 -4
- data/src/core/lib/event_engine/posix_engine/timer.h +5 -7
- data/src/core/lib/event_engine/posix_engine/timer_heap.cc +2 -3
- data/src/core/lib/event_engine/posix_engine/timer_heap.h +2 -2
- data/src/core/lib/event_engine/posix_engine/timer_manager.cc +3 -4
- data/src/core/lib/event_engine/posix_engine/timer_manager.h +5 -7
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.cc +5 -7
- data/src/core/lib/event_engine/posix_engine/traced_buffer_list.h +3 -5
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.cc +5 -6
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_eventfd.h +2 -3
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.cc +4 -5
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_pipe.h +2 -3
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix.h +2 -2
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.cc +2 -3
- data/src/core/lib/event_engine/posix_engine/wakeup_fd_posix_default.h +2 -2
- data/src/core/lib/event_engine/query_extensions.h +2 -2
- data/src/core/lib/event_engine/ref_counted_dns_resolver_interface.h +3 -5
- data/src/core/lib/event_engine/resolved_address.cc +5 -4
- data/src/core/lib/event_engine/slice.cc +3 -5
- data/src/core/lib/event_engine/slice_buffer.cc +1 -2
- data/src/core/lib/event_engine/tcp_socket_utils.cc +3 -10
- data/src/core/lib/event_engine/tcp_socket_utils.h +3 -3
- data/src/core/lib/event_engine/thread_pool/thread_count.cc +2 -4
- data/src/core/lib/event_engine/thread_pool/thread_count.h +5 -6
- data/src/core/lib/event_engine/thread_pool/thread_pool.h +2 -4
- data/src/core/lib/event_engine/thread_pool/thread_pool_factory.cc +2 -3
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.cc +9 -11
- data/src/core/lib/event_engine/thread_pool/work_stealing_thread_pool.h +7 -9
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.cc +5 -5
- data/src/core/lib/event_engine/thready_event_engine/thready_event_engine.h +5 -5
- data/src/core/lib/event_engine/time_util.cc +2 -2
- data/src/core/lib/event_engine/time_util.h +1 -2
- data/src/core/lib/event_engine/utils.cc +19 -5
- data/src/core/lib/event_engine/utils.h +10 -4
- data/src/core/lib/event_engine/windows/grpc_polled_fd_windows.cc +5 -8
- data/src/core/lib/event_engine/windows/grpc_polled_fd_windows.h +4 -6
- data/src/core/lib/event_engine/windows/iocp.cc +4 -5
- data/src/core/lib/event_engine/windows/iocp.h +1 -2
- data/src/core/lib/event_engine/windows/native_windows_dns_resolver.cc +4 -6
- data/src/core/lib/event_engine/windows/win_socket.cc +4 -5
- data/src/core/lib/event_engine/windows/win_socket.h +4 -5
- data/src/core/lib/event_engine/windows/windows_endpoint.cc +28 -15
- data/src/core/lib/event_engine/windows/windows_engine.cc +10 -11
- data/src/core/lib/event_engine/windows/windows_engine.h +8 -9
- data/src/core/lib/event_engine/windows/windows_listener.cc +2 -3
- data/src/core/lib/event_engine/windows/windows_listener.h +4 -5
- data/src/core/lib/event_engine/work_queue/basic_work_queue.cc +3 -3
- data/src/core/lib/event_engine/work_queue/basic_work_queue.h +3 -5
- data/src/core/lib/event_engine/work_queue/work_queue.h +2 -3
- data/src/core/lib/experiments/config.cc +9 -12
- data/src/core/lib/experiments/config.h +1 -2
- data/src/core/lib/experiments/experiments.cc +203 -28
- data/src/core/lib/experiments/experiments.h +76 -15
- data/src/core/lib/iomgr/buffer_list.cc +4 -5
- data/src/core/lib/iomgr/buffer_list.h +2 -3
- data/src/core/lib/iomgr/call_combiner.cc +2 -4
- data/src/core/lib/iomgr/call_combiner.h +6 -8
- data/src/core/lib/iomgr/cfstream_handle.cc +2 -4
- data/src/core/lib/iomgr/cfstream_handle.h +1 -1
- data/src/core/lib/iomgr/closure.cc +2 -2
- data/src/core/lib/iomgr/closure.h +9 -9
- data/src/core/lib/iomgr/combiner.cc +4 -6
- data/src/core/lib/iomgr/combiner.h +2 -3
- data/src/core/lib/iomgr/endpoint.h +1 -2
- data/src/core/lib/iomgr/endpoint_cfstream.cc +2 -4
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -4
- data/src/core/lib/iomgr/endpoint_pair_windows.cc +1 -2
- data/src/core/lib/iomgr/error.cc +6 -8
- data/src/core/lib/iomgr/error.h +6 -8
- data/src/core/lib/iomgr/error_cfstream.cc +1 -3
- data/src/core/lib/iomgr/ev_apple.cc +3 -4
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +7 -9
- data/src/core/lib/iomgr/ev_poll_posix.cc +3 -5
- data/src/core/lib/iomgr/ev_posix.cc +4 -6
- data/src/core/lib/iomgr/ev_posix.h +1 -2
- data/src/core/lib/iomgr/event_engine_shims/closure.cc +3 -4
- data/src/core/lib/iomgr/event_engine_shims/closure.h +1 -2
- data/src/core/lib/iomgr/event_engine_shims/endpoint.cc +10 -11
- data/src/core/lib/iomgr/event_engine_shims/endpoint.h +2 -2
- data/src/core/lib/iomgr/event_engine_shims/tcp_client.cc +2 -3
- data/src/core/lib/iomgr/exec_ctx.cc +4 -5
- data/src/core/lib/iomgr/exec_ctx.h +6 -8
- data/src/core/lib/iomgr/executor.cc +6 -8
- data/src/core/lib/iomgr/executor.h +1 -1
- data/src/core/lib/iomgr/fork_posix.cc +5 -7
- data/src/core/lib/iomgr/fork_windows.cc +2 -2
- data/src/core/lib/iomgr/internal_errqueue.cc +2 -3
- data/src/core/lib/iomgr/iocp_windows.cc +4 -6
- data/src/core/lib/iomgr/iomgr.cc +7 -9
- data/src/core/lib/iomgr/iomgr.h +1 -2
- data/src/core/lib/iomgr/iomgr_internal.cc +1 -2
- data/src/core/lib/iomgr/iomgr_internal.h +1 -2
- data/src/core/lib/iomgr/iomgr_posix.cc +2 -1
- data/src/core/lib/iomgr/iomgr_windows.cc +4 -3
- data/src/core/lib/iomgr/lockfree_event.cc +3 -4
- data/src/core/lib/iomgr/nameser.h +1 -1
- data/src/core/lib/iomgr/polling_entity.cc +3 -4
- data/src/core/lib/iomgr/pollset_set_windows.cc +1 -2
- data/src/core/lib/iomgr/pollset_windows.cc +2 -2
- data/src/core/lib/iomgr/port.h +2 -2
- data/src/core/lib/iomgr/resolve_address.cc +3 -4
- data/src/core/lib/iomgr/resolve_address.h +4 -6
- data/src/core/lib/iomgr/resolve_address_impl.h +1 -2
- data/src/core/lib/iomgr/resolve_address_posix.cc +6 -7
- data/src/core/lib/iomgr/resolve_address_posix.h +2 -2
- data/src/core/lib/iomgr/resolve_address_windows.cc +8 -10
- data/src/core/lib/iomgr/resolve_address_windows.h +2 -2
- data/src/core/lib/iomgr/resolved_address.h +1 -2
- data/src/core/lib/iomgr/sockaddr_utils_posix.cc +3 -5
- data/src/core/lib/iomgr/socket_mutator.cc +1 -1
- data/src/core/lib/iomgr/socket_mutator.h +1 -2
- data/src/core/lib/iomgr/socket_utils.h +1 -2
- data/src/core/lib/iomgr/socket_utils_common_posix.cc +5 -7
- data/src/core/lib/iomgr/socket_utils_linux.cc +1 -1
- data/src/core/lib/iomgr/socket_utils_posix.cc +4 -6
- data/src/core/lib/iomgr/socket_utils_windows.cc +1 -1
- data/src/core/lib/iomgr/socket_windows.cc +3 -5
- data/src/core/lib/iomgr/socket_windows.h +1 -2
- data/src/core/lib/iomgr/tcp_client_cfstream.cc +5 -7
- data/src/core/lib/iomgr/tcp_client_posix.cc +5 -7
- data/src/core/lib/iomgr/tcp_client_windows.cc +6 -8
- data/src/core/lib/iomgr/tcp_posix.cc +14 -17
- data/src/core/lib/iomgr/tcp_posix.h +1 -1
- data/src/core/lib/iomgr/tcp_server.h +2 -2
- data/src/core/lib/iomgr/tcp_server_posix.cc +9 -11
- data/src/core/lib/iomgr/tcp_server_utils_posix.h +2 -3
- data/src/core/lib/iomgr/tcp_server_utils_posix_common.cc +3 -5
- data/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc +4 -6
- data/src/core/lib/iomgr/tcp_server_windows.cc +11 -13
- data/src/core/lib/iomgr/tcp_windows.cc +4 -6
- data/src/core/lib/iomgr/timer.h +2 -2
- data/src/core/lib/iomgr/timer_generic.cc +9 -11
- data/src/core/lib/iomgr/timer_heap.cc +2 -3
- data/src/core/lib/iomgr/timer_manager.cc +4 -6
- data/src/core/lib/iomgr/timer_manager.h +1 -2
- data/src/core/lib/iomgr/unix_sockets_posix.cc +3 -4
- data/src/core/lib/iomgr/unix_sockets_posix.h +3 -4
- data/src/core/lib/iomgr/unix_sockets_posix_noop.cc +0 -2
- data/src/core/lib/iomgr/vsock.cc +2 -4
- data/src/core/lib/iomgr/vsock.h +3 -4
- data/src/core/lib/iomgr/wakeup_fd_eventfd.cc +2 -2
- data/src/core/lib/iomgr/wakeup_fd_pipe.cc +3 -4
- data/src/core/lib/promise/activity.cc +2 -4
- data/src/core/lib/promise/activity.h +10 -12
- data/src/core/lib/promise/all_ok.h +40 -4
- data/src/core/lib/promise/arena_promise.h +2 -4
- data/src/core/lib/promise/cancel_callback.h +4 -4
- data/src/core/lib/promise/context.h +7 -8
- data/src/core/lib/promise/detail/basic_seq.h +33 -19
- data/src/core/lib/promise/detail/join_state.h +4 -5
- data/src/core/lib/promise/detail/promise_factory.h +7 -8
- data/src/core/lib/promise/detail/promise_like.h +26 -4
- data/src/core/lib/promise/detail/seq_state.h +183 -185
- data/src/core/lib/promise/detail/status.h +5 -5
- data/src/core/lib/promise/exec_ctx_wakeup_scheduler.h +2 -3
- data/src/core/lib/promise/for_each.h +9 -10
- data/src/core/lib/promise/if.h +9 -9
- data/src/core/lib/promise/interceptor_list.h +3 -5
- data/src/core/lib/promise/latch.h +1 -3
- data/src/core/lib/promise/loop.h +13 -5
- data/src/core/lib/promise/map.h +57 -7
- data/src/core/lib/promise/observable.h +3 -4
- data/src/core/lib/promise/party.cc +42 -76
- data/src/core/lib/promise/party.h +60 -40
- data/src/core/lib/promise/pipe.h +3 -5
- data/src/core/lib/promise/poll.h +19 -7
- data/src/core/lib/promise/prioritized_race.h +2 -2
- data/src/core/lib/promise/promise.h +5 -6
- data/src/core/lib/promise/race.h +4 -4
- data/src/core/lib/promise/seq.h +17 -39
- data/src/core/lib/promise/sleep.cc +3 -3
- data/src/core/lib/promise/sleep.h +4 -5
- data/src/core/lib/promise/status_flag.h +9 -3
- data/src/core/lib/promise/try_join.h +5 -5
- data/src/core/lib/promise/try_seq.h +45 -47
- data/src/core/lib/resource_quota/api.cc +4 -6
- data/src/core/lib/resource_quota/api.h +2 -3
- data/src/core/lib/resource_quota/arena.cc +3 -4
- data/src/core/lib/resource_quota/arena.h +4 -5
- data/src/core/lib/resource_quota/connection_quota.cc +2 -2
- data/src/core/lib/resource_quota/connection_quota.h +5 -6
- data/src/core/lib/resource_quota/memory_quota.cc +20 -11
- data/src/core/lib/resource_quota/memory_quota.h +21 -13
- data/src/core/lib/resource_quota/periodic_update.cc +4 -4
- data/src/core/lib/resource_quota/periodic_update.h +2 -4
- data/src/core/lib/resource_quota/resource_quota.h +7 -8
- data/src/core/lib/resource_quota/thread_quota.cc +2 -2
- data/src/core/lib/resource_quota/thread_quota.h +5 -6
- data/src/core/lib/security/authorization/audit_logging.cc +5 -6
- data/src/core/lib/security/authorization/audit_logging.h +5 -6
- data/src/core/lib/security/authorization/authorization_engine.h +3 -3
- data/src/core/lib/security/authorization/authorization_policy_provider.h +3 -4
- data/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc +1 -1
- data/src/core/lib/security/authorization/evaluate_args.cc +4 -6
- data/src/core/lib/security/authorization/evaluate_args.h +3 -4
- data/src/core/lib/security/authorization/grpc_authorization_engine.cc +2 -3
- data/src/core/lib/security/authorization/grpc_authorization_engine.h +2 -3
- data/src/core/lib/security/authorization/grpc_server_authz_filter.cc +5 -3
- data/src/core/lib/security/authorization/grpc_server_authz_filter.h +2 -3
- data/src/core/lib/security/authorization/matchers.cc +2 -4
- data/src/core/lib/security/authorization/matchers.h +2 -4
- data/src/core/lib/security/authorization/rbac_policy.cc +2 -2
- data/src/core/lib/security/authorization/rbac_policy.h +3 -5
- data/src/core/lib/security/authorization/stdout_logger.cc +4 -4
- data/src/core/lib/security/authorization/stdout_logger.h +4 -4
- data/src/core/lib/security/certificate_provider/certificate_provider_factory.h +6 -7
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.cc +2 -2
- data/src/core/lib/security/certificate_provider/certificate_provider_registry.h +2 -3
- data/src/core/lib/security/context/security_context.cc +6 -8
- data/src/core/lib/security/context/security_context.h +16 -10
- data/src/core/lib/security/credentials/alts/alts_credentials.cc +5 -5
- data/src/core/lib/security/credentials/alts/alts_credentials.h +8 -4
- data/src/core/lib/security/credentials/alts/check_gcp_environment.cc +2 -3
- data/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc +1 -2
- data/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc +1 -2
- data/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc +3 -4
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc +1 -2
- data/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc +2 -2
- data/src/core/lib/security/credentials/call_creds_util.cc +6 -5
- data/src/core/lib/security/credentials/call_creds_util.h +2 -2
- data/src/core/lib/security/credentials/channel_creds_registry.h +5 -6
- data/src/core/lib/security/credentials/channel_creds_registry_init.cc +10 -11
- data/src/core/lib/security/credentials/composite/composite_credentials.cc +3 -4
- data/src/core/lib/security/credentials/composite/composite_credentials.h +8 -9
- data/src/core/lib/security/credentials/credentials.cc +1 -4
- data/src/core/lib/security/credentials/credentials.h +11 -12
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.cc +10 -12
- data/src/core/lib/security/credentials/external/aws_external_account_credentials.h +7 -6
- data/src/core/lib/security/credentials/external/aws_request_signer.cc +4 -5
- data/src/core/lib/security/credentials/external/aws_request_signer.h +3 -3
- data/src/core/lib/security/credentials/external/external_account_credentials.cc +15 -16
- data/src/core/lib/security/credentials/external/external_account_credentials.h +5 -7
- data/src/core/lib/security/credentials/external/file_external_account_credentials.cc +6 -7
- data/src/core/lib/security/credentials/external/file_external_account_credentials.h +6 -5
- data/src/core/lib/security/credentials/external/url_external_account_credentials.cc +8 -10
- data/src/core/lib/security/credentials/external/url_external_account_credentials.h +8 -7
- data/src/core/lib/security/credentials/fake/fake_credentials.cc +2 -4
- data/src/core/lib/security/credentials/fake/fake_credentials.h +6 -7
- data/src/core/lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.cc +5 -6
- data/src/core/lib/security/credentials/gcp_service_account_identity/gcp_service_account_identity_credentials.h +7 -7
- data/src/core/lib/security/credentials/google_default/credentials_generic.cc +3 -4
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +18 -21
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +7 -5
- data/src/core/lib/security/credentials/iam/iam_credentials.cc +2 -4
- data/src/core/lib/security/credentials/iam/iam_credentials.h +5 -6
- data/src/core/lib/security/credentials/insecure/insecure_credentials.cc +2 -2
- data/src/core/lib/security/credentials/insecure/insecure_credentials.h +2 -2
- data/src/core/lib/security/credentials/jwt/json_token.cc +11 -14
- data/src/core/lib/security/credentials/jwt/json_token.h +1 -2
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +8 -10
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +7 -9
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +19 -22
- data/src/core/lib/security/credentials/jwt/jwt_verifier.h +2 -3
- data/src/core/lib/security/credentials/local/local_credentials.cc +2 -2
- data/src/core/lib/security/credentials/local/local_credentials.h +2 -2
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +14 -16
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +12 -13
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +3 -4
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +9 -11
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +4 -6
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +4 -6
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.cc +3 -3
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h +4 -5
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_match.cc +1 -3
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.cc +79 -9
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h +13 -11
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.cc +4 -6
- data/src/core/lib/security/credentials/tls/grpc_tls_certificate_verifier.h +8 -9
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc +4 -5
- data/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h +1 -1
- data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.cc +3 -5
- data/src/core/lib/security/credentials/tls/grpc_tls_crl_provider.h +8 -10
- data/src/core/lib/security/credentials/tls/tls_credentials.cc +6 -7
- data/src/core/lib/security/credentials/tls/tls_credentials.h +5 -3
- data/src/core/lib/security/credentials/tls/tls_utils.cc +1 -2
- data/src/core/lib/security/credentials/tls/tls_utils.h +3 -3
- data/src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.cc +29 -22
- data/src/core/lib/security/credentials/token_fetcher/token_fetcher_credentials.h +15 -10
- data/src/core/lib/security/credentials/xds/xds_credentials.cc +2 -3
- data/src/core/lib/security/credentials/xds/xds_credentials.h +7 -9
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +9 -11
- data/src/core/lib/security/security_connector/alts/alts_security_connector.h +1 -1
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +9 -11
- data/src/core/lib/security/security_connector/fake/fake_security_connector.h +1 -1
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.cc +4 -6
- data/src/core/lib/security/security_connector/insecure/insecure_security_connector.h +5 -6
- data/src/core/lib/security/security_connector/load_system_roots_supported.cc +5 -7
- data/src/core/lib/security/security_connector/load_system_roots_windows.cc +3 -4
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +17 -13
- data/src/core/lib/security/security_connector/local/local_security_connector.h +1 -1
- data/src/core/lib/security/security_connector/security_connector.cc +2 -4
- data/src/core/lib/security/security_connector/security_connector.h +7 -8
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +6 -8
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +2 -3
- data/src/core/lib/security/security_connector/ssl_utils.cc +12 -14
- data/src/core/lib/security/security_connector/ssl_utils.h +7 -9
- data/src/core/lib/security/security_connector/tls/tls_security_connector.cc +8 -10
- data/src/core/lib/security/security_connector/tls/tls_security_connector.h +7 -8
- data/src/core/lib/security/transport/auth_filters.h +2 -3
- data/src/core/lib/security/transport/client_auth_filter.cc +8 -10
- data/src/core/lib/security/transport/server_auth_filter.cc +10 -11
- data/src/core/lib/security/util/json_util.cc +4 -5
- data/src/core/lib/slice/percent_encoding.cc +2 -4
- data/src/core/lib/slice/slice.cc +4 -6
- data/src/core/lib/slice/slice.h +5 -7
- data/src/core/lib/slice/slice_buffer.cc +5 -6
- data/src/core/lib/slice/slice_buffer.h +3 -4
- data/src/core/lib/slice/slice_internal.h +3 -5
- data/src/core/lib/slice/slice_refcount.h +2 -3
- data/src/core/lib/slice/slice_string_helpers.h +1 -2
- data/src/core/lib/surface/byte_buffer.cc +1 -2
- data/src/core/lib/surface/byte_buffer_reader.cc +3 -5
- data/src/core/lib/surface/call.cc +22 -24
- data/src/core/lib/surface/call.h +6 -8
- data/src/core/lib/surface/call_log_batch.cc +3 -5
- data/src/core/lib/surface/call_test_only.h +1 -2
- data/src/core/lib/surface/call_utils.cc +16 -59
- data/src/core/lib/surface/call_utils.h +57 -19
- data/src/core/lib/surface/channel.cc +1 -2
- data/src/core/lib/surface/channel.h +11 -12
- data/src/core/lib/surface/channel_create.cc +2 -3
- data/src/core/lib/surface/channel_create.h +2 -3
- data/src/core/lib/surface/channel_init.cc +4 -6
- data/src/core/lib/surface/channel_init.h +4 -6
- data/src/core/lib/surface/client_call.cc +66 -54
- data/src/core/lib/surface/client_call.h +27 -19
- data/src/core/lib/surface/completion_queue.cc +12 -14
- data/src/core/lib/surface/completion_queue.h +4 -5
- data/src/core/lib/surface/completion_queue_factory.cc +1 -2
- data/src/core/lib/surface/connection_context.cc +77 -0
- data/src/core/lib/surface/connection_context.h +156 -0
- data/src/core/lib/surface/event_string.cc +2 -2
- data/src/core/lib/surface/event_string.h +2 -2
- data/src/core/lib/surface/filter_stack_call.cc +18 -20
- data/src/core/lib/surface/filter_stack_call.h +16 -18
- data/src/core/lib/surface/init.cc +28 -7
- data/src/core/lib/surface/init.h +5 -0
- data/src/core/lib/surface/lame_client.cc +10 -11
- data/src/core/lib/surface/lame_client.h +4 -5
- data/src/core/lib/surface/legacy_channel.cc +11 -12
- data/src/core/lib/surface/legacy_channel.h +6 -7
- data/src/core/lib/surface/metadata_array.cc +1 -2
- data/src/core/lib/surface/server_call.cc +22 -18
- data/src/core/lib/surface/server_call.h +19 -20
- data/src/core/lib/surface/validate_metadata.cc +3 -4
- data/src/core/lib/surface/validate_metadata.h +2 -4
- data/src/core/lib/surface/version.cc +2 -2
- data/src/core/lib/transport/bdp_estimator.cc +2 -3
- data/src/core/lib/transport/bdp_estimator.h +3 -5
- data/src/core/lib/transport/call_arena_allocator.cc +2 -2
- data/src/core/lib/transport/call_arena_allocator.h +2 -3
- data/src/core/lib/transport/call_destination.h +1 -1
- data/src/core/lib/transport/call_filters.cc +5 -117
- data/src/core/lib/transport/call_filters.h +402 -35
- data/src/core/lib/transport/call_final_info.cc +2 -2
- data/src/core/lib/transport/call_final_info.h +3 -3
- data/src/core/lib/transport/call_spine.cc +1 -6
- data/src/core/lib/transport/call_spine.h +29 -12
- data/src/core/lib/transport/call_state.h +145 -41
- data/src/core/lib/transport/connectivity_state.cc +3 -4
- data/src/core/lib/transport/connectivity_state.h +5 -6
- data/src/core/lib/transport/error_utils.cc +3 -4
- data/src/core/lib/transport/error_utils.h +4 -5
- data/src/core/lib/transport/interception_chain.cc +3 -3
- data/src/core/lib/transport/interception_chain.h +17 -8
- data/src/core/lib/transport/message.cc +2 -2
- data/src/core/lib/transport/message.h +5 -0
- data/src/core/lib/transport/metadata_batch.cc +12 -5
- data/src/core/lib/transport/metadata_batch.h +17 -11
- data/src/core/lib/transport/metadata_compression_traits.h +3 -4
- data/src/core/lib/transport/parsed_metadata.h +3 -5
- data/src/core/lib/transport/simple_slice_based_metadata.h +1 -2
- data/src/core/lib/transport/status_conversion.h +1 -1
- data/src/core/lib/transport/timeout_encoding.cc +7 -7
- data/src/core/lib/transport/timeout_encoding.h +2 -4
- data/src/core/lib/transport/transport.cc +4 -6
- data/src/core/lib/transport/transport.h +7 -9
- data/src/core/lib/transport/transport_framing_endpoint_extension.h +47 -0
- data/src/core/lib/transport/transport_op_string.cc +4 -5
- data/src/core/load_balancing/address_filtering.cc +2 -4
- data/src/core/load_balancing/address_filtering.h +4 -5
- data/src/core/load_balancing/backend_metric_data.h +2 -2
- data/src/core/load_balancing/backend_metric_parser.cc +1 -2
- data/src/core/load_balancing/backend_metric_parser.h +1 -3
- data/src/core/load_balancing/child_policy_handler.cc +5 -6
- data/src/core/load_balancing/child_policy_handler.h +4 -5
- data/src/core/load_balancing/delegating_helper.h +6 -7
- data/src/core/load_balancing/endpoint_list.cc +7 -9
- data/src/core/load_balancing/endpoint_list.h +6 -8
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.cc +10 -4
- data/src/core/load_balancing/grpclb/client_load_reporting_filter.h +1 -2
- data/src/core/load_balancing/grpclb/grpclb.cc +20 -22
- data/src/core/load_balancing/grpclb/grpclb_balancer_addresses.cc +1 -2
- data/src/core/load_balancing/grpclb/grpclb_client_stats.cc +2 -3
- data/src/core/load_balancing/grpclb/grpclb_client_stats.h +5 -7
- data/src/core/load_balancing/grpclb/load_balancer_api.cc +4 -6
- data/src/core/load_balancing/grpclb/load_balancer_api.h +4 -6
- data/src/core/load_balancing/health_check_client.cc +20 -20
- data/src/core/load_balancing/health_check_client.h +3 -3
- data/src/core/load_balancing/health_check_client_internal.h +8 -9
- data/src/core/load_balancing/lb_policy.h +13 -15
- data/src/core/load_balancing/lb_policy_factory.h +4 -5
- data/src/core/load_balancing/lb_policy_registry.cc +3 -4
- data/src/core/load_balancing/lb_policy_registry.h +4 -5
- data/src/core/load_balancing/oob_backend_metric.cc +14 -16
- data/src/core/load_balancing/oob_backend_metric.h +3 -3
- data/src/core/load_balancing/oob_backend_metric_internal.h +8 -9
- data/src/core/load_balancing/outlier_detection/outlier_detection.cc +31 -26
- data/src/core/load_balancing/outlier_detection/outlier_detection.h +3 -5
- data/src/core/load_balancing/pick_first/pick_first.cc +28 -25
- data/src/core/load_balancing/priority/priority.cc +12 -14
- data/src/core/load_balancing/ring_hash/ring_hash.cc +13 -15
- data/src/core/load_balancing/ring_hash/ring_hash.h +3 -4
- data/src/core/load_balancing/rls/rls.cc +27 -29
- data/src/core/load_balancing/round_robin/round_robin.cc +9 -15
- data/src/core/load_balancing/subchannel_interface.h +13 -6
- data/src/core/load_balancing/weighted_round_robin/static_stride_scheduler.cc +4 -4
- data/src/core/load_balancing/weighted_round_robin/static_stride_scheduler.h +2 -3
- data/src/core/load_balancing/weighted_round_robin/weighted_round_robin.cc +15 -19
- data/src/core/load_balancing/weighted_target/weighted_target.cc +11 -13
- data/src/core/load_balancing/xds/cds.cc +23 -20
- data/src/core/load_balancing/xds/xds_cluster_impl.cc +42 -45
- data/src/core/load_balancing/xds/xds_cluster_manager.cc +10 -12
- data/src/core/load_balancing/xds/xds_override_host.cc +15 -17
- data/src/core/load_balancing/xds/xds_override_host.h +3 -4
- data/src/core/load_balancing/xds/xds_wrr_locality.cc +11 -12
- data/src/core/plugin_registry/grpc_plugin_registry.cc +1 -7
- data/src/core/plugin_registry/grpc_plugin_registry_extra.cc +3 -1
- data/src/core/resolver/dns/c_ares/dns_resolver_ares.cc +13 -16
- data/src/core/resolver/dns/c_ares/dns_resolver_ares.h +2 -3
- data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver.h +4 -6
- data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +2 -4
- data/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +8 -11
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc +15 -19
- data/src/core/resolver/dns/c_ares/grpc_ares_wrapper.h +3 -6
- data/src/core/resolver/dns/dns_resolver_plugin.cc +4 -5
- data/src/core/resolver/dns/dns_resolver_plugin.h +1 -1
- data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc +9 -11
- data/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.h +3 -4
- data/src/core/resolver/dns/event_engine/service_config_helper.cc +3 -5
- data/src/core/resolver/dns/event_engine/service_config_helper.h +2 -2
- data/src/core/resolver/dns/native/dns_resolver.cc +10 -11
- data/src/core/resolver/dns/native/dns_resolver.h +1 -1
- data/src/core/resolver/endpoint_addresses.cc +1 -3
- data/src/core/resolver/endpoint_addresses.h +2 -3
- data/src/core/resolver/fake/fake_resolver.cc +7 -8
- data/src/core/resolver/fake/fake_resolver.h +7 -8
- data/src/core/resolver/google_c2p/google_c2p_resolver.cc +11 -12
- data/src/core/resolver/polling_resolver.cc +6 -8
- data/src/core/resolver/polling_resolver.h +7 -8
- data/src/core/resolver/resolver.h +4 -5
- data/src/core/resolver/resolver_factory.h +4 -5
- data/src/core/resolver/resolver_registry.cc +2 -2
- data/src/core/resolver/resolver_registry.h +4 -5
- data/src/core/resolver/sockaddr/sockaddr_resolver.cc +5 -6
- data/src/core/resolver/xds/xds_config.cc +1 -2
- data/src/core/resolver/xds/xds_config.h +1 -2
- data/src/core/resolver/xds/xds_dependency_manager.cc +2 -5
- data/src/core/resolver/xds/xds_dependency_manager.h +1 -2
- data/src/core/resolver/xds/xds_resolver.cc +15 -17
- data/src/core/resolver/xds/xds_resolver_attributes.h +2 -3
- data/src/core/server/server.cc +97 -74
- data/src/core/server/server.h +13 -16
- data/src/core/server/server_call_tracer_filter.cc +11 -4
- data/src/core/server/server_call_tracer_filter.h +1 -1
- data/src/core/server/server_config_selector.h +5 -6
- data/src/core/server/server_config_selector_filter.cc +8 -6
- data/src/core/server/xds_channel_stack_modifier.cc +3 -3
- data/src/core/server/xds_channel_stack_modifier.h +5 -6
- data/src/core/server/xds_server_config_fetcher.cc +23 -24
- data/src/core/service_config/service_config.h +3 -5
- data/src/core/service_config/service_config_call_data.h +5 -6
- data/src/core/service_config/service_config_channel_arg_filter.cc +8 -6
- data/src/core/service_config/service_config_impl.cc +4 -6
- data/src/core/service_config/service_config_impl.h +4 -6
- data/src/core/service_config/service_config_parser.cc +1 -2
- data/src/core/service_config/service_config_parser.h +2 -4
- data/src/core/telemetry/call_tracer.cc +2 -3
- data/src/core/telemetry/call_tracer.h +5 -6
- data/src/core/telemetry/histogram_view.cc +3 -3
- data/src/core/telemetry/histogram_view.h +1 -2
- data/src/core/telemetry/metrics.cc +18 -16
- data/src/core/telemetry/metrics.h +13 -10
- data/src/core/telemetry/stats.cc +1 -2
- data/src/core/telemetry/stats.h +2 -4
- data/src/core/telemetry/stats_data.cc +114 -2
- data/src/core/telemetry/stats_data.h +82 -4
- data/src/core/telemetry/tcp_tracer.h +1 -2
- data/src/core/tsi/alts/crypt/aes_gcm.cc +5 -8
- data/src/core/tsi/alts/crypt/gsec.cc +2 -3
- data/src/core/tsi/alts/crypt/gsec.h +3 -4
- data/src/core/tsi/alts/frame_protector/alts_counter.cc +1 -2
- data/src/core/tsi/alts/frame_protector/alts_counter.h +3 -4
- data/src/core/tsi/alts/frame_protector/alts_crypter.cc +1 -2
- data/src/core/tsi/alts/frame_protector/alts_crypter.h +2 -3
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.cc +4 -5
- data/src/core/tsi/alts/frame_protector/alts_frame_protector.h +1 -2
- data/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc +1 -1
- data/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc +1 -1
- data/src/core/tsi/alts/frame_protector/frame_handler.cc +4 -6
- data/src/core/tsi/alts/frame_protector/frame_handler.h +1 -2
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +11 -12
- data/src/core/tsi/alts/handshaker/alts_shared_resource.cc +2 -3
- data/src/core/tsi/alts/handshaker/alts_shared_resource.h +1 -1
- data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +19 -17
- data/src/core/tsi/alts/handshaker/alts_tsi_utils.cc +2 -3
- data/src/core/tsi/alts/handshaker/transport_security_common_api.cc +2 -2
- data/src/core/tsi/alts/handshaker/transport_security_common_api.h +3 -3
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc +3 -5
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h +1 -2
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc +2 -3
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.h +1 -2
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc +3 -5
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc +3 -4
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.h +1 -2
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc +3 -5
- data/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h +1 -2
- data/src/core/tsi/fake_transport_security.cc +4 -7
- data/src/core/tsi/local_transport_security.cc +4 -6
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.cc +4 -5
- data/src/core/tsi/ssl/key_logging/ssl_key_logging.h +8 -10
- data/src/core/tsi/ssl/session_cache/ssl_session.h +4 -5
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +4 -5
- data/src/core/tsi/ssl/session_cache/ssl_session_cache.h +7 -8
- data/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc +2 -4
- data/src/core/tsi/ssl_transport_security.cc +29 -17
- data/src/core/tsi/ssl_transport_security.h +6 -8
- data/src/core/tsi/ssl_transport_security_utils.cc +1 -3
- data/src/core/tsi/ssl_transport_security_utils.h +3 -5
- data/src/core/tsi/ssl_types.h +1 -2
- data/src/core/tsi/transport_security.cc +2 -3
- data/src/core/tsi/transport_security.h +1 -2
- data/src/core/tsi/transport_security_interface.h +3 -4
- data/src/core/util/alloc.cc +2 -5
- data/src/core/util/atm.cc +1 -2
- data/src/core/{lib/gprpp → util}/atomic_utils.h +5 -5
- data/src/core/{lib/avl → util}/avl.h +6 -7
- data/src/core/{lib/backoff → util}/backoff.cc +3 -1
- data/src/core/{lib/backoff → util}/backoff.h +4 -5
- data/src/core/{lib/gprpp → util}/bitset.h +4 -5
- data/src/core/{lib/gprpp → util}/chunked_vector.h +6 -7
- data/src/core/{lib/gprpp → util}/construct_destruct.h +8 -7
- data/src/core/{lib/gprpp → util}/cpp_impl_of.h +3 -3
- data/src/core/{lib/gprpp → util}/crash.cc +2 -3
- data/src/core/{lib/gprpp → util}/crash.h +5 -6
- data/src/core/{lib/gprpp → util}/debug_location.h +5 -5
- data/src/core/{lib/gprpp → util}/directory_reader.h +5 -5
- data/src/core/{lib/gprpp → util}/down_cast.h +7 -7
- data/src/core/{lib/gprpp → util}/dual_ref_counted.h +10 -11
- data/src/core/{lib/gprpp → util}/dump_args.cc +1 -1
- data/src/core/{lib/gprpp → util}/dump_args.h +12 -7
- data/src/core/{lib/gprpp → util}/env.h +5 -5
- data/src/core/{lib/debug → util}/event_log.cc +3 -3
- data/src/core/{lib/debug → util}/event_log.h +6 -8
- data/src/core/{lib/gprpp → util}/examine_stack.cc +1 -1
- data/src/core/{lib/gprpp → util}/examine_stack.h +5 -5
- data/src/core/{lib/gprpp → util}/fork.cc +5 -5
- data/src/core/{lib/gprpp → util}/fork.h +7 -7
- data/src/core/util/gcp_metadata_query.cc +8 -11
- data/src/core/util/gcp_metadata_query.h +2 -3
- data/src/core/{lib/iomgr → util}/gethostname.h +3 -3
- data/src/core/{lib/iomgr → util}/gethostname_fallback.cc +1 -1
- data/src/core/{lib/iomgr → util}/gethostname_host_name_max.cc +2 -3
- data/src/core/{lib/iomgr → util}/gethostname_sysconf.cc +2 -3
- data/src/core/{lib/gprpp → util}/glob.h +3 -3
- data/src/core/util/gpr_time.cc +268 -0
- data/src/core/{lib/iomgr → util}/grpc_if_nametoindex.h +4 -5
- data/src/core/{lib/iomgr → util}/grpc_if_nametoindex_posix.cc +2 -3
- data/src/core/{lib/iomgr → util}/grpc_if_nametoindex_unsupported.cc +2 -3
- data/src/core/{lib/gprpp → util}/host_port.cc +2 -3
- data/src/core/{lib/gprpp → util}/host_port.h +5 -5
- data/src/core/util/http_client/format_request.cc +2 -5
- data/src/core/util/http_client/format_request.h +1 -2
- data/src/core/util/http_client/httpcli.cc +19 -15
- data/src/core/util/http_client/httpcli.h +8 -11
- data/src/core/util/http_client/httpcli_security_connector.cc +9 -12
- data/src/core/util/http_client/httpcli_ssl_credentials.h +2 -3
- data/src/core/util/http_client/parser.cc +2 -4
- data/src/core/util/http_client/parser.h +1 -3
- data/src/core/{lib/gprpp → util}/if_list.h +3 -3
- data/src/core/util/iphone/cpu.cc +1 -2
- data/src/core/util/json/json.h +1 -2
- data/src/core/util/json/json_channel_args.h +0 -1
- data/src/core/util/json/json_object_loader.cc +3 -4
- data/src/core/util/json/json_object_loader.h +8 -9
- data/src/core/util/json/json_reader.cc +2 -5
- data/src/core/util/json/json_reader.h +0 -1
- data/src/core/util/json/json_util.cc +4 -4
- data/src/core/util/json/json_util.h +1 -2
- data/src/core/util/json/json_writer.cc +1 -3
- data/src/core/util/latent_see.cc +99 -69
- data/src/core/util/latent_see.h +45 -20
- data/src/core/util/linux/cpu.cc +4 -6
- data/src/core/{lib/gprpp → util}/linux/env.cc +3 -3
- data/src/core/{lib/gprpp → util}/load_file.cc +4 -5
- data/src/core/{lib/gprpp → util}/load_file.h +5 -6
- data/src/core/util/log.cc +10 -13
- data/src/core/util/lru_cache.h +27 -9
- data/src/core/{lib/gprpp → util}/manual_constructor.h +5 -6
- data/src/core/{lib/gprpp → util}/match.h +6 -7
- data/src/core/{lib/matchers → util}/matchers.cc +10 -3
- data/src/core/{lib/matchers → util}/matchers.h +9 -5
- data/src/core/{lib/gprpp → util}/memory.h +6 -6
- data/src/core/{lib/gprpp → util}/mpscq.cc +1 -1
- data/src/core/{lib/gprpp → util}/mpscq.h +6 -7
- data/src/core/util/msys/tmpfile.cc +3 -4
- data/src/core/{lib/gprpp → util}/no_destruct.h +6 -6
- data/src/core/{lib/gprpp → util}/notification.h +6 -7
- data/src/core/{lib/gprpp → util}/orphanable.h +9 -9
- data/src/core/{lib/gprpp → util}/overload.h +5 -5
- data/src/core/{lib/gprpp → util}/packed_table.h +5 -5
- data/src/core/{lib/gprpp → util}/per_cpu.cc +1 -1
- data/src/core/{lib/gprpp → util}/per_cpu.h +5 -6
- data/src/core/util/posix/cpu.cc +3 -5
- data/src/core/{lib/gprpp → util}/posix/directory_reader.cc +4 -4
- data/src/core/{lib/gprpp → util}/posix/env.cc +1 -1
- data/src/core/{lib/gprpp → util}/posix/stat.cc +4 -5
- data/src/core/util/posix/string.cc +2 -3
- data/src/core/util/posix/sync.cc +3 -5
- data/src/core/{lib/gprpp → util}/posix/thd.cc +8 -11
- data/src/core/util/posix/time.cc +4 -5
- data/src/core/util/posix/tmpfile.cc +4 -6
- data/src/core/{lib/backoff → util}/random_early_detection.cc +3 -3
- data/src/core/{lib/backoff → util}/random_early_detection.h +4 -5
- data/src/core/{lib/gprpp → util}/ref_counted.h +9 -10
- data/src/core/{lib/gprpp → util}/ref_counted_ptr.h +6 -8
- data/src/core/{lib/gprpp → util}/ref_counted_string.cc +3 -4
- data/src/core/{lib/gprpp → util}/ref_counted_string.h +6 -8
- data/src/core/{lib/gprpp → util}/single_set_ptr.h +5 -5
- data/src/core/{lib/gprpp → util}/sorted_pack.h +6 -6
- data/src/core/util/spinlock.h +1 -2
- data/src/core/{lib/gprpp → util}/stat.h +4 -5
- data/src/core/{lib/gprpp → util}/status_helper.cc +5 -7
- data/src/core/{lib/gprpp → util}/status_helper.h +5 -7
- data/src/core/{lib/gprpp → util}/strerror.cc +2 -3
- data/src/core/{lib/gprpp → util}/strerror.h +5 -5
- data/src/core/util/string.cc +6 -9
- data/src/core/util/string.h +1 -3
- data/src/core/util/sync.cc +3 -5
- data/src/core/{lib/gprpp → util}/sync.h +7 -7
- data/src/core/util/sync_abseil.cc +5 -7
- data/src/core/{lib/gprpp → util}/table.h +6 -8
- data/src/core/{lib/gprpp → util}/tchar.cc +1 -1
- data/src/core/{lib/gprpp → util}/tchar.h +3 -3
- data/src/core/{lib/gprpp → util}/thd.h +5 -6
- data/src/core/util/time.cc +174 -204
- data/src/core/{lib/gprpp → util}/time.h +16 -10
- data/src/core/{lib/gprpp → util}/time_averaged_stats.cc +1 -1
- data/src/core/{lib/gprpp → util}/time_averaged_stats.h +3 -3
- data/src/core/util/time_precise.cc +3 -4
- data/src/core/util/time_precise.h +0 -1
- data/src/core/{lib/gprpp → util}/time_util.cc +3 -4
- data/src/core/{lib/gprpp → util}/time_util.h +5 -5
- data/src/core/util/tmpfile.h +0 -1
- data/src/core/{lib/gprpp → util}/type_list.h +3 -3
- data/src/core/{lib/gprpp → util}/unique_type_name.h +5 -6
- data/src/core/{lib/uri/uri_parser.cc → util/uri.cc} +12 -7
- data/src/core/{lib/uri/uri_parser.h → util/uri.h} +10 -6
- data/src/core/{lib/gprpp → util}/uuid_v4.cc +3 -3
- data/src/core/{lib/gprpp → util}/uuid_v4.h +4 -5
- data/src/core/{lib/gprpp → util}/validation_errors.cc +2 -3
- data/src/core/{lib/gprpp → util}/validation_errors.h +4 -5
- data/src/core/util/windows/cpu.cc +1 -1
- data/src/core/{lib/gprpp → util}/windows/directory_reader.cc +1 -1
- data/src/core/{lib/gprpp → util}/windows/env.cc +2 -2
- data/src/core/{lib/gprpp → util}/windows/stat.cc +3 -4
- data/src/core/util/windows/string.cc +2 -3
- data/src/core/util/windows/string_util.cc +5 -7
- data/src/core/util/windows/sync.cc +2 -3
- data/src/core/{lib/gprpp → util}/windows/thd.cc +6 -8
- data/src/core/util/windows/time.cc +2 -4
- data/src/core/util/windows/tmpfile.cc +4 -5
- data/src/core/{lib/gprpp → util}/work_serializer.cc +7 -12
- data/src/core/{lib/gprpp → util}/work_serializer.h +8 -9
- data/src/core/{lib/gprpp → util}/xxhash_inline.h +3 -3
- data/src/core/xds/grpc/certificate_provider_store.cc +3 -4
- data/src/core/xds/grpc/certificate_provider_store.h +9 -10
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc +4 -5
- data/src/core/xds/grpc/file_watcher_certificate_provider_factory.h +6 -7
- data/src/core/xds/grpc/xds_audit_logger_registry.cc +4 -5
- data/src/core/xds/grpc/xds_audit_logger_registry.h +3 -4
- data/src/core/xds/grpc/xds_bootstrap_grpc.cc +4 -5
- data/src/core/xds/grpc/xds_bootstrap_grpc.h +4 -4
- data/src/core/xds/grpc/xds_certificate_provider.cc +2 -3
- data/src/core/xds/grpc/xds_certificate_provider.h +8 -9
- data/src/core/xds/grpc/xds_client_grpc.cc +63 -39
- data/src/core/xds/grpc/xds_client_grpc.h +14 -8
- data/src/core/xds/grpc/xds_cluster.cc +9 -4
- data/src/core/xds/grpc/xds_cluster.h +30 -4
- data/src/core/xds/grpc/xds_cluster_parser.cc +106 -26
- data/src/core/xds/grpc/xds_cluster_parser.h +6 -2
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.cc +5 -7
- data/src/core/xds/grpc/xds_cluster_specifier_plugin.h +5 -6
- data/src/core/xds/grpc/xds_common_types.cc +1 -2
- data/src/core/xds/grpc/xds_common_types.h +2 -3
- data/src/core/xds/grpc/xds_common_types_parser.cc +45 -9
- data/src/core/xds/grpc/xds_common_types_parser.h +8 -3
- data/src/core/xds/grpc/xds_endpoint.h +7 -5
- data/src/core/xds/grpc/xds_endpoint_parser.cc +45 -40
- data/src/core/xds/grpc/xds_endpoint_parser.h +1 -2
- data/src/core/xds/grpc/xds_health_status.h +0 -1
- data/src/core/xds/grpc/xds_http_fault_filter.cc +5 -7
- data/src/core/xds/grpc/xds_http_fault_filter.h +4 -5
- data/src/core/xds/grpc/xds_http_filter.h +2 -3
- data/src/core/xds/grpc/xds_http_filter_registry.cc +2 -3
- data/src/core/xds/grpc/xds_http_filter_registry.h +2 -3
- data/src/core/xds/grpc/xds_http_gcp_authn_filter.cc +5 -6
- data/src/core/xds/grpc/xds_http_gcp_authn_filter.h +2 -3
- data/src/core/xds/grpc/xds_http_rbac_filter.cc +42 -39
- data/src/core/xds/grpc/xds_http_rbac_filter.h +4 -5
- data/src/core/xds/grpc/xds_http_stateful_session_filter.cc +5 -6
- data/src/core/xds/grpc/xds_http_stateful_session_filter.h +4 -5
- data/src/core/xds/grpc/xds_lb_policy_registry.cc +5 -7
- data/src/core/xds/grpc/xds_lb_policy_registry.h +4 -5
- data/src/core/xds/grpc/xds_listener.cc +1 -2
- data/src/core/xds/grpc/xds_listener.h +1 -2
- data/src/core/xds/grpc/xds_listener_parser.cc +5 -8
- data/src/core/xds/grpc/xds_listener_parser.h +1 -2
- data/src/core/xds/grpc/xds_metadata.h +26 -3
- data/src/core/xds/grpc/xds_metadata_parser.cc +51 -10
- data/src/core/xds/grpc/xds_metadata_parser.h +1 -2
- data/src/core/xds/grpc/xds_route_config.cc +3 -3
- data/src/core/xds/grpc/xds_route_config.h +2 -3
- data/src/core/xds/grpc/xds_route_config_parser.cc +11 -13
- data/src/core/xds/grpc/xds_route_config_parser.h +4 -7
- data/src/core/xds/grpc/xds_routing.cc +2 -4
- data/src/core/xds/grpc/xds_routing.h +1 -3
- data/src/core/xds/grpc/xds_server_grpc.cc +1 -2
- data/src/core/xds/grpc/xds_server_grpc.h +2 -2
- data/src/core/xds/grpc/xds_transport_grpc.cc +95 -50
- data/src/core/xds/grpc/xds_transport_grpc.h +33 -22
- data/src/core/xds/xds_client/lrs_client.cc +1292 -0
- data/src/core/xds/xds_client/lrs_client.h +394 -0
- data/src/core/xds/xds_client/xds_api.cc +31 -245
- data/src/core/xds/xds_client/xds_api.h +13 -36
- data/src/core/xds/xds_client/xds_backend_metric_propagation.cc +63 -0
- data/src/core/xds/xds_client/xds_backend_metric_propagation.h +59 -0
- data/src/core/xds/xds_client/xds_bootstrap.cc +2 -3
- data/src/core/xds/xds_client/xds_bootstrap.h +2 -2
- data/src/core/xds/xds_client/xds_client.cc +57 -595
- data/src/core/xds/xds_client/xds_client.h +33 -86
- data/src/core/xds/xds_client/xds_locality.h +102 -0
- data/src/core/xds/xds_client/xds_metrics.h +2 -2
- data/src/core/xds/xds_client/xds_resource_type.h +4 -5
- data/src/core/xds/xds_client/xds_resource_type_impl.h +3 -4
- data/src/core/xds/xds_client/xds_transport.h +29 -14
- data/src/ruby/ext/grpc/extconf.rb +1 -1
- data/src/ruby/ext/grpc/rb_byte_buffer.c +3 -3
- data/src/ruby/ext/grpc/rb_call.c +5 -5
- data/src/ruby/ext/grpc/rb_call_credentials.c +5 -6
- data/src/ruby/ext/grpc/rb_channel.c +6 -7
- data/src/ruby/ext/grpc/rb_channel_args.c +3 -3
- data/src/ruby/ext/grpc/rb_channel_credentials.c +5 -6
- data/src/ruby/ext/grpc/rb_channel_credentials.h +1 -2
- data/src/ruby/ext/grpc/rb_completion_queue.c +3 -4
- data/src/ruby/ext/grpc/rb_compression_options.c +6 -7
- data/src/ruby/ext/grpc/rb_event_thread.c +4 -5
- data/src/ruby/ext/grpc/rb_grpc.c +3 -4
- data/src/ruby/ext/grpc/rb_grpc.h +2 -3
- data/src/ruby/ext/grpc/rb_server.c +6 -6
- data/src/ruby/ext/grpc/rb_server_credentials.c +3 -3
- data/src/ruby/ext/grpc/rb_server_credentials.h +1 -2
- data/src/ruby/ext/grpc/rb_xds_channel_credentials.c +5 -6
- data/src/ruby/ext/grpc/rb_xds_channel_credentials.h +1 -2
- data/src/ruby/ext/grpc/rb_xds_server_credentials.c +4 -4
- data/src/ruby/ext/grpc/rb_xds_server_credentials.h +1 -2
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/src/ruby/spec/generic/client_stub_spec.rb +20 -20
- data/third_party/abseil-cpp/absl/algorithm/container.h +57 -18
- data/third_party/abseil-cpp/absl/base/attributes.h +84 -0
- data/third_party/abseil-cpp/absl/base/config.h +32 -51
- data/third_party/abseil-cpp/absl/base/dynamic_annotations.h +0 -16
- data/third_party/abseil-cpp/absl/base/internal/nullability_impl.h +3 -1
- data/third_party/abseil-cpp/absl/base/internal/spinlock.h +13 -3
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock.cc +0 -12
- data/third_party/abseil-cpp/absl/base/internal/unscaledcycleclock_config.h +4 -4
- data/third_party/abseil-cpp/absl/base/macros.h +48 -0
- data/third_party/abseil-cpp/absl/base/no_destructor.h +35 -40
- data/third_party/abseil-cpp/absl/base/nullability.h +33 -7
- data/third_party/abseil-cpp/absl/base/optimization.h +11 -0
- data/third_party/abseil-cpp/absl/base/options.h +1 -1
- data/third_party/abseil-cpp/absl/base/prefetch.h +1 -1
- data/third_party/abseil-cpp/absl/container/flat_hash_map.h +68 -12
- data/third_party/abseil-cpp/absl/container/flat_hash_set.h +60 -6
- data/third_party/abseil-cpp/absl/container/hash_container_defaults.h +45 -0
- data/third_party/abseil-cpp/absl/container/inlined_vector.h +13 -0
- data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h +20 -11
- data/third_party/abseil-cpp/absl/container/internal/compressed_tuple.h +15 -16
- data/third_party/abseil-cpp/absl/container/internal/container_memory.h +34 -1
- data/third_party/abseil-cpp/absl/container/internal/hash_function_defaults.h +68 -1
- data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h +50 -0
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc +22 -7
- data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h +26 -8
- data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h +39 -35
- data/third_party/abseil-cpp/absl/container/internal/layout.h +190 -74
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_map.h +8 -6
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc +334 -71
- data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.h +1299 -458
- data/third_party/abseil-cpp/absl/crc/internal/crc32_x86_arm_combined_simd.h +19 -17
- data/third_party/abseil-cpp/absl/crc/internal/crc_cord_state.cc +4 -3
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc +2 -1
- data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_x86_arm_combined.cc +12 -8
- data/third_party/abseil-cpp/absl/crc/internal/crc_non_temporal_memcpy.cc +1 -1
- data/third_party/abseil-cpp/absl/crc/internal/crc_x86_arm_combined.cc +11 -7
- data/third_party/abseil-cpp/absl/crc/internal/non_temporal_memcpy.h +18 -17
- data/third_party/abseil-cpp/absl/debugging/internal/bounded_utf8_length_sequence.h +126 -0
- data/third_party/abseil-cpp/absl/debugging/internal/decode_rust_punycode.cc +258 -0
- data/third_party/abseil-cpp/absl/debugging/internal/decode_rust_punycode.h +55 -0
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.cc +1057 -86
- data/third_party/abseil-cpp/absl/debugging/internal/demangle.h +3 -0
- data/third_party/abseil-cpp/absl/debugging/internal/demangle_rust.cc +925 -0
- data/third_party/abseil-cpp/absl/debugging/internal/demangle_rust.h +42 -0
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.cc +43 -16
- data/third_party/abseil-cpp/absl/debugging/internal/elf_mem_image.h +8 -7
- data/third_party/abseil-cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc +10 -7
- data/third_party/abseil-cpp/absl/debugging/internal/utf8_for_code_point.cc +70 -0
- data/third_party/abseil-cpp/absl/debugging/internal/utf8_for_code_point.h +47 -0
- data/third_party/abseil-cpp/absl/flags/commandlineflag.h +11 -0
- data/third_party/abseil-cpp/absl/flags/flag.h +2 -0
- data/third_party/abseil-cpp/absl/flags/internal/flag.cc +117 -30
- data/third_party/abseil-cpp/absl/flags/internal/flag.h +192 -30
- data/third_party/abseil-cpp/absl/flags/reflection.cc +10 -0
- data/third_party/abseil-cpp/absl/functional/any_invocable.h +13 -3
- data/third_party/abseil-cpp/absl/functional/bind_front.h +3 -2
- data/third_party/abseil-cpp/absl/functional/internal/any_invocable.h +7 -7
- data/third_party/abseil-cpp/absl/functional/internal/front_binder.h +9 -9
- data/third_party/abseil-cpp/absl/hash/internal/hash.cc +1 -1
- data/third_party/abseil-cpp/absl/hash/internal/hash.h +13 -3
- data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.cc +60 -28
- data/third_party/abseil-cpp/absl/hash/internal/low_level_hash.h +4 -0
- data/third_party/abseil-cpp/absl/log/absl_vlog_is_on.h +3 -3
- data/third_party/abseil-cpp/absl/log/globals.h +28 -15
- data/third_party/abseil-cpp/absl/log/internal/check_op.cc +20 -0
- data/third_party/abseil-cpp/absl/log/internal/check_op.h +63 -21
- data/third_party/abseil-cpp/absl/log/internal/conditions.h +2 -2
- data/third_party/abseil-cpp/absl/log/internal/log_impl.h +23 -23
- data/third_party/abseil-cpp/absl/log/internal/log_message.cc +104 -47
- data/third_party/abseil-cpp/absl/log/internal/log_message.h +23 -4
- data/third_party/abseil-cpp/absl/log/internal/nullstream.h +1 -10
- data/third_party/abseil-cpp/absl/log/internal/strip.h +36 -0
- data/third_party/abseil-cpp/absl/log/log.h +5 -1
- data/third_party/abseil-cpp/absl/log/log_sink.h +11 -4
- data/third_party/abseil-cpp/absl/log/vlog_is_on.h +3 -3
- data/third_party/abseil-cpp/absl/meta/type_traits.h +138 -42
- data/third_party/abseil-cpp/absl/numeric/int128.cc +0 -3
- data/third_party/abseil-cpp/absl/numeric/int128.h +35 -5
- data/third_party/abseil-cpp/absl/numeric/int128_have_intrinsic.inc +14 -0
- data/third_party/abseil-cpp/absl/numeric/int128_no_intrinsic.inc +18 -0
- data/third_party/abseil-cpp/absl/numeric/internal/bits.h +6 -2
- data/third_party/abseil-cpp/absl/random/beta_distribution.h +8 -8
- data/third_party/abseil-cpp/absl/random/bit_gen_ref.h +9 -7
- data/third_party/abseil-cpp/absl/random/distributions.h +11 -11
- data/third_party/abseil-cpp/absl/random/seed_sequences.h +2 -0
- data/third_party/abseil-cpp/absl/status/internal/statusor_internal.h +61 -2
- data/third_party/abseil-cpp/absl/status/status.cc +0 -4
- data/third_party/abseil-cpp/absl/status/status.h +4 -4
- data/third_party/abseil-cpp/absl/status/statusor.h +108 -142
- data/third_party/abseil-cpp/absl/strings/ascii.cc +32 -71
- data/third_party/abseil-cpp/absl/strings/cord.cc +20 -15
- data/third_party/abseil-cpp/absl/strings/cord.h +68 -7
- data/third_party/abseil-cpp/absl/strings/escaping.cc +96 -21
- data/third_party/abseil-cpp/absl/strings/escaping.h +25 -8
- data/third_party/abseil-cpp/absl/strings/has_absl_stringify.h +1 -0
- data/third_party/abseil-cpp/absl/strings/internal/charconv_bigint.h +10 -0
- data/third_party/abseil-cpp/absl/strings/internal/cord_internal.h +29 -9
- data/third_party/abseil-cpp/absl/strings/internal/cord_rep_btree.h +2 -2
- data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.cc +19 -13
- data/third_party/abseil-cpp/absl/strings/internal/cordz_functions.h +23 -13
- data/third_party/abseil-cpp/absl/strings/internal/cordz_handle.cc +24 -24
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.cc +12 -7
- data/third_party/abseil-cpp/absl/strings/internal/cordz_info.h +9 -4
- data/third_party/abseil-cpp/absl/strings/internal/escaping.cc +5 -2
- data/third_party/abseil-cpp/absl/strings/internal/str_join_internal.h +23 -2
- data/third_party/abseil-cpp/absl/strings/internal/str_split_internal.h +5 -1
- data/third_party/abseil-cpp/absl/strings/numbers.cc +107 -333
- data/third_party/abseil-cpp/absl/strings/numbers.h +12 -151
- data/third_party/abseil-cpp/absl/strings/str_cat.cc +49 -142
- data/third_party/abseil-cpp/absl/strings/str_cat.h +70 -85
- data/third_party/abseil-cpp/absl/strings/str_format.h +1 -1
- data/third_party/abseil-cpp/absl/strings/str_join.h +19 -5
- data/third_party/abseil-cpp/absl/strings/str_split.h +2 -2
- data/third_party/abseil-cpp/absl/strings/string_view.h +3 -2
- data/third_party/abseil-cpp/absl/strings/substitute.cc +4 -0
- data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.cc +11 -2
- data/third_party/abseil-cpp/absl/synchronization/internal/graphcycles.h +5 -0
- data/third_party/abseil-cpp/absl/synchronization/mutex.h +5 -4
- data/third_party/abseil-cpp/absl/time/civil_time.h +2 -2
- data/third_party/abseil-cpp/absl/time/clock.cc +15 -1
- data/third_party/abseil-cpp/absl/time/duration.cc +58 -53
- data/third_party/abseil-cpp/absl/time/format.cc +2 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_libc.cc +1 -1
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/time_zone_lookup.cc +0 -29
- data/third_party/abseil-cpp/absl/time/internal/cctz/src/tzfile.h +3 -3
- data/third_party/abseil-cpp/absl/time/time.h +73 -29
- data/third_party/abseil-cpp/absl/types/compare.h +505 -0
- data/third_party/abseil-cpp/absl/types/internal/optional.h +2 -2
- data/third_party/abseil-cpp/absl/types/internal/variant.h +55 -67
- data/third_party/abseil-cpp/absl/types/optional.h +15 -18
- data/third_party/abseil-cpp/absl/types/span.h +3 -2
- data/third_party/abseil-cpp/absl/types/variant.h +19 -24
- data/third_party/abseil-cpp/absl/utility/utility.h +3 -41
- data/third_party/upb/upb/json/decode.c +59 -13
- data/third_party/upb/upb/json/decode.h +26 -3
- data/third_party/upb/upb/json/encode.c +2 -2
- data/third_party/upb/upb/lex/round_trip.c +10 -0
- data/third_party/upb/upb/mem/arena.c +13 -2
- data/third_party/upb/upb/mem/arena.h +9 -0
- data/third_party/upb/upb/mem/arena.hpp +5 -1
- data/third_party/upb/upb/message/accessors.c +2 -4
- data/third_party/upb/upb/message/accessors.h +178 -379
- data/third_party/upb/upb/message/copy.c +11 -7
- data/third_party/upb/upb/message/internal/accessors.h +578 -26
- data/third_party/upb/upb/message/internal/extension.c +1 -0
- data/third_party/upb/upb/message/internal/message.c +13 -14
- data/third_party/upb/upb/message/internal/message.h +8 -7
- data/third_party/upb/upb/message/merge.c +38 -0
- data/third_party/upb/upb/message/merge.h +26 -0
- data/third_party/upb/upb/message/message.h +6 -10
- data/third_party/upb/upb/message/value.h +26 -0
- data/third_party/upb/upb/mini_descriptor/decode.c +7 -2
- data/third_party/upb/upb/mini_descriptor/link.c +7 -3
- data/third_party/upb/upb/mini_table/extension_registry.c +18 -0
- data/third_party/upb/upb/mini_table/extension_registry.h +17 -0
- data/third_party/upb/upb/mini_table/internal/extension.h +9 -0
- data/third_party/upb/upb/mini_table/internal/message.c +21 -1
- data/third_party/upb/upb/mini_table/internal/message.h +40 -20
- data/third_party/upb/upb/mini_table/internal/sub.h +5 -0
- data/third_party/upb/upb/mini_table/message.h +11 -1
- data/third_party/upb/upb/port/def.inc +99 -9
- data/third_party/upb/upb/port/undef.inc +7 -0
- data/third_party/upb/upb/reflection/common.h +1 -11
- data/third_party/upb/upb/reflection/def.hpp +8 -0
- data/third_party/upb/upb/reflection/descriptor_bootstrap.h +19 -0
- data/third_party/upb/upb/reflection/field_def.c +11 -22
- data/third_party/upb/upb/reflection/field_def.h +1 -0
- data/third_party/upb/upb/reflection/internal/upb_edition_defaults.h +1 -1
- data/third_party/upb/upb/reflection/message.c +8 -3
- data/third_party/upb/upb/reflection/message.h +2 -2
- data/third_party/upb/upb/text/encode.c +51 -394
- data/third_party/upb/upb/text/encode.h +1 -11
- data/third_party/upb/upb/text/internal/encode.c +180 -0
- data/third_party/upb/upb/text/internal/encode.h +240 -0
- data/third_party/upb/upb/text/options.h +22 -0
- data/third_party/upb/upb/wire/decode.c +71 -39
- data/third_party/upb/upb/wire/decode.h +4 -1
- data/third_party/upb/upb/wire/encode.c +41 -18
- data/third_party/upb/upb/wire/encode.h +3 -0
- data/third_party/upb/upb/wire/internal/decode_fast.c +3 -0
- metadata +142 -109
- data/src/core/lib/gprpp/time.cc +0 -241
- data/src/core/resolver/binder/binder_resolver.cc +0 -153
- data/src/core/xds/xds_client/xds_client_stats.cc +0 -164
- data/src/core/xds/xds_client/xds_client_stats.h +0 -258
- data/third_party/abseil-cpp/absl/strings/internal/has_absl_stringify.h +0 -44
- /data/src/core/{lib/gprpp → util}/glob.cc +0 -0
@@ -80,7 +80,7 @@
|
|
80
80
|
// slot_type slots[capacity];
|
81
81
|
// };
|
82
82
|
//
|
83
|
-
// The length of this array is computed by `
|
83
|
+
// The length of this array is computed by `RawHashSetLayout::alloc_size` below.
|
84
84
|
//
|
85
85
|
// Control bytes (`ctrl_t`) are bytes (collected into groups of a
|
86
86
|
// platform-specific size) that define the state of the corresponding slot in
|
@@ -100,6 +100,13 @@
|
|
100
100
|
// Storing control bytes in a separate array also has beneficial cache effects,
|
101
101
|
// since more logical slots will fit into a cache line.
|
102
102
|
//
|
103
|
+
// # Small Object Optimization (SOO)
|
104
|
+
//
|
105
|
+
// When the size/alignment of the value_type and the capacity of the table are
|
106
|
+
// small, we enable small object optimization and store the values inline in
|
107
|
+
// the raw_hash_set object. This optimization allows us to avoid
|
108
|
+
// allocation/deallocation as well as cache/dTLB misses.
|
109
|
+
//
|
103
110
|
// # Hashing
|
104
111
|
//
|
105
112
|
// We compute two separate hashes, `H1` and `H2`, from the hash of an object.
|
@@ -233,9 +240,10 @@ namespace container_internal {
|
|
233
240
|
|
234
241
|
#ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS
|
235
242
|
#error ABSL_SWISSTABLE_ENABLE_GENERATIONS cannot be directly set
|
236
|
-
#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) ||
|
237
|
-
|
238
|
-
|
243
|
+
#elif (defined(ABSL_HAVE_ADDRESS_SANITIZER) || \
|
244
|
+
defined(ABSL_HAVE_HWADDRESS_SANITIZER) || \
|
245
|
+
defined(ABSL_HAVE_MEMORY_SANITIZER)) && \
|
246
|
+
!defined(NDEBUG_SANITIZER) // If defined, performance is important.
|
239
247
|
// When compiled in sanitizer mode, we add generation integers to the backing
|
240
248
|
// array and iterators. In the backing array, we store the generation between
|
241
249
|
// the control bytes and the slots. When iterators are dereferenced, we assert
|
@@ -374,6 +382,9 @@ uint32_t TrailingZeros(T x) {
|
|
374
382
|
return static_cast<uint32_t>(countr_zero(x));
|
375
383
|
}
|
376
384
|
|
385
|
+
// 8 bytes bitmask with most significant bit set for every byte.
|
386
|
+
constexpr uint64_t kMsbs8Bytes = 0x8080808080808080ULL;
|
387
|
+
|
377
388
|
// An abstract bitmask, such as that emitted by a SIMD instruction.
|
378
389
|
//
|
379
390
|
// Specifically, this type implements a simple bitset whose representation is
|
@@ -423,27 +434,35 @@ class NonIterableBitMask {
|
|
423
434
|
// an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When
|
424
435
|
// `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as
|
425
436
|
// the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask.
|
437
|
+
// If NullifyBitsOnIteration is true (only allowed for Shift == 3),
|
438
|
+
// non zero abstract bit is allowed to have additional bits
|
439
|
+
// (e.g., `0xff`, `0x83` and `0x9c` are ok, but `0x6f` is not).
|
426
440
|
//
|
427
441
|
// For example:
|
428
442
|
// for (int i : BitMask<uint32_t, 16>(0b101)) -> yields 0, 2
|
429
443
|
// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3
|
430
|
-
template <class T, int SignificantBits, int Shift = 0
|
444
|
+
template <class T, int SignificantBits, int Shift = 0,
|
445
|
+
bool NullifyBitsOnIteration = false>
|
431
446
|
class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> {
|
432
447
|
using Base = NonIterableBitMask<T, SignificantBits, Shift>;
|
433
448
|
static_assert(std::is_unsigned<T>::value, "");
|
434
449
|
static_assert(Shift == 0 || Shift == 3, "");
|
450
|
+
static_assert(!NullifyBitsOnIteration || Shift == 3, "");
|
435
451
|
|
436
452
|
public:
|
437
|
-
explicit BitMask(T mask) : Base(mask) {
|
453
|
+
explicit BitMask(T mask) : Base(mask) {
|
454
|
+
if (Shift == 3 && !NullifyBitsOnIteration) {
|
455
|
+
assert(this->mask_ == (this->mask_ & kMsbs8Bytes));
|
456
|
+
}
|
457
|
+
}
|
438
458
|
// BitMask is an iterator over the indices of its abstract bits.
|
439
459
|
using value_type = int;
|
440
460
|
using iterator = BitMask;
|
441
461
|
using const_iterator = BitMask;
|
442
462
|
|
443
463
|
BitMask& operator++() {
|
444
|
-
if (Shift == 3) {
|
445
|
-
|
446
|
-
this->mask_ &= msbs;
|
464
|
+
if (Shift == 3 && NullifyBitsOnIteration) {
|
465
|
+
this->mask_ &= kMsbs8Bytes;
|
447
466
|
}
|
448
467
|
this->mask_ &= (this->mask_ - 1);
|
449
468
|
return *this;
|
@@ -520,10 +539,24 @@ ABSL_DLL extern const ctrl_t kEmptyGroup[32];
|
|
520
539
|
// Returns a pointer to a control byte group that can be used by empty tables.
|
521
540
|
inline ctrl_t* EmptyGroup() {
|
522
541
|
// Const must be cast away here; no uses of this function will actually write
|
523
|
-
// to it
|
542
|
+
// to it because it is only used for empty tables.
|
524
543
|
return const_cast<ctrl_t*>(kEmptyGroup + 16);
|
525
544
|
}
|
526
545
|
|
546
|
+
// For use in SOO iterators.
|
547
|
+
// TODO(b/289225379): we could potentially get rid of this by adding an is_soo
|
548
|
+
// bit in iterators. This would add branches but reduce cache misses.
|
549
|
+
ABSL_DLL extern const ctrl_t kSooControl[17];
|
550
|
+
|
551
|
+
// Returns a pointer to a full byte followed by a sentinel byte.
|
552
|
+
inline ctrl_t* SooControl() {
|
553
|
+
// Const must be cast away here; no uses of this function will actually write
|
554
|
+
// to it because it is only used for SOO iterators.
|
555
|
+
return const_cast<ctrl_t*>(kSooControl);
|
556
|
+
}
|
557
|
+
// Whether ctrl is from the SooControl array.
|
558
|
+
inline bool IsSooControl(const ctrl_t* ctrl) { return ctrl == SooControl(); }
|
559
|
+
|
527
560
|
// Returns a pointer to a generation to use for an empty hashtable.
|
528
561
|
GenerationType* EmptyGeneration();
|
529
562
|
|
@@ -535,7 +568,37 @@ inline bool IsEmptyGeneration(const GenerationType* generation) {
|
|
535
568
|
|
536
569
|
// Mixes a randomly generated per-process seed with `hash` and `ctrl` to
|
537
570
|
// randomize insertion order within groups.
|
538
|
-
bool
|
571
|
+
bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash,
|
572
|
+
const ctrl_t* ctrl);
|
573
|
+
|
574
|
+
ABSL_ATTRIBUTE_ALWAYS_INLINE inline bool ShouldInsertBackwards(
|
575
|
+
ABSL_ATTRIBUTE_UNUSED size_t capacity, ABSL_ATTRIBUTE_UNUSED size_t hash,
|
576
|
+
ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) {
|
577
|
+
#if defined(NDEBUG)
|
578
|
+
return false;
|
579
|
+
#else
|
580
|
+
return ShouldInsertBackwardsForDebug(capacity, hash, ctrl);
|
581
|
+
#endif
|
582
|
+
}
|
583
|
+
|
584
|
+
// Returns insert position for the given mask.
|
585
|
+
// We want to add entropy even when ASLR is not enabled.
|
586
|
+
// In debug build we will randomly insert in either the front or back of
|
587
|
+
// the group.
|
588
|
+
// TODO(kfm,sbenza): revisit after we do unconditional mixing
|
589
|
+
template <class Mask>
|
590
|
+
ABSL_ATTRIBUTE_ALWAYS_INLINE inline auto GetInsertionOffset(
|
591
|
+
Mask mask, ABSL_ATTRIBUTE_UNUSED size_t capacity,
|
592
|
+
ABSL_ATTRIBUTE_UNUSED size_t hash,
|
593
|
+
ABSL_ATTRIBUTE_UNUSED const ctrl_t* ctrl) {
|
594
|
+
#if defined(NDEBUG)
|
595
|
+
return mask.LowestBitSet();
|
596
|
+
#else
|
597
|
+
return ShouldInsertBackwardsForDebug(capacity, hash, ctrl)
|
598
|
+
? mask.HighestBitSet()
|
599
|
+
: mask.LowestBitSet();
|
600
|
+
#endif
|
601
|
+
}
|
539
602
|
|
540
603
|
// Returns a per-table, hash salt, which changes on resize. This gets mixed into
|
541
604
|
// H1 to randomize iteration order per-table.
|
@@ -560,7 +623,12 @@ inline h2_t H2(size_t hash) { return hash & 0x7F; }
|
|
560
623
|
|
561
624
|
// Helpers for checking the state of a control byte.
|
562
625
|
inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; }
|
563
|
-
inline bool IsFull(ctrl_t c) {
|
626
|
+
inline bool IsFull(ctrl_t c) {
|
627
|
+
// Cast `c` to the underlying type instead of casting `0` to `ctrl_t` as `0`
|
628
|
+
// is not a value in the enum. Both ways are equivalent, but this way makes
|
629
|
+
// linters happier.
|
630
|
+
return static_cast<std::underlying_type_t<ctrl_t>>(c) >= 0;
|
631
|
+
}
|
564
632
|
inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; }
|
565
633
|
inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; }
|
566
634
|
|
@@ -646,6 +714,14 @@ struct GroupSse2Impl {
|
|
646
714
|
static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff));
|
647
715
|
}
|
648
716
|
|
717
|
+
// Returns a bitmask representing the positions of non full slots.
|
718
|
+
// Note: this includes: kEmpty, kDeleted, kSentinel.
|
719
|
+
// It is useful in contexts when kSentinel is not present.
|
720
|
+
auto MaskNonFull() const {
|
721
|
+
return BitMask<uint16_t, kWidth>(
|
722
|
+
static_cast<uint16_t>(_mm_movemask_epi8(ctrl)));
|
723
|
+
}
|
724
|
+
|
649
725
|
// Returns a bitmask representing the positions of empty or deleted slots.
|
650
726
|
NonIterableBitMask<uint16_t, kWidth> MaskEmptyOrDeleted() const {
|
651
727
|
auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
|
@@ -685,10 +761,11 @@ struct GroupAArch64Impl {
|
|
685
761
|
ctrl = vld1_u8(reinterpret_cast<const uint8_t*>(pos));
|
686
762
|
}
|
687
763
|
|
688
|
-
|
764
|
+
auto Match(h2_t hash) const {
|
689
765
|
uint8x8_t dup = vdup_n_u8(hash);
|
690
766
|
auto mask = vceq_u8(ctrl, dup);
|
691
|
-
return BitMask<uint64_t, kWidth, 3
|
767
|
+
return BitMask<uint64_t, kWidth, /*Shift=*/3,
|
768
|
+
/*NullifyBitsOnIteration=*/true>(
|
692
769
|
vget_lane_u64(vreinterpret_u64_u8(mask), 0));
|
693
770
|
}
|
694
771
|
|
@@ -704,12 +781,25 @@ struct GroupAArch64Impl {
|
|
704
781
|
// Returns a bitmask representing the positions of full slots.
|
705
782
|
// Note: for `is_small()` tables group may contain the "same" slot twice:
|
706
783
|
// original and mirrored.
|
707
|
-
|
784
|
+
auto MaskFull() const {
|
708
785
|
uint64_t mask = vget_lane_u64(
|
709
786
|
vreinterpret_u64_u8(vcge_s8(vreinterpret_s8_u8(ctrl),
|
710
787
|
vdup_n_s8(static_cast<int8_t>(0)))),
|
711
788
|
0);
|
712
|
-
return BitMask<uint64_t, kWidth, 3
|
789
|
+
return BitMask<uint64_t, kWidth, /*Shift=*/3,
|
790
|
+
/*NullifyBitsOnIteration=*/true>(mask);
|
791
|
+
}
|
792
|
+
|
793
|
+
// Returns a bitmask representing the positions of non full slots.
|
794
|
+
// Note: this includes: kEmpty, kDeleted, kSentinel.
|
795
|
+
// It is useful in contexts when kSentinel is not present.
|
796
|
+
auto MaskNonFull() const {
|
797
|
+
uint64_t mask = vget_lane_u64(
|
798
|
+
vreinterpret_u64_u8(vclt_s8(vreinterpret_s8_u8(ctrl),
|
799
|
+
vdup_n_s8(static_cast<int8_t>(0)))),
|
800
|
+
0);
|
801
|
+
return BitMask<uint64_t, kWidth, /*Shift=*/3,
|
802
|
+
/*NullifyBitsOnIteration=*/true>(mask);
|
713
803
|
}
|
714
804
|
|
715
805
|
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
|
@@ -736,11 +826,10 @@ struct GroupAArch64Impl {
|
|
736
826
|
|
737
827
|
void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
|
738
828
|
uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0);
|
739
|
-
constexpr uint64_t msbs = 0x8080808080808080ULL;
|
740
829
|
constexpr uint64_t slsbs = 0x0202020202020202ULL;
|
741
830
|
constexpr uint64_t midbs = 0x7e7e7e7e7e7e7e7eULL;
|
742
831
|
auto x = slsbs & (mask >> 6);
|
743
|
-
auto res = (x + midbs) |
|
832
|
+
auto res = (x + midbs) | kMsbs8Bytes;
|
744
833
|
little_endian::Store64(dst, res);
|
745
834
|
}
|
746
835
|
|
@@ -768,30 +857,33 @@ struct GroupPortableImpl {
|
|
768
857
|
// v = 0x1716151413121110
|
769
858
|
// hash = 0x12
|
770
859
|
// retval = (v - lsbs) & ~v & msbs = 0x0000000080800000
|
771
|
-
constexpr uint64_t msbs = 0x8080808080808080ULL;
|
772
860
|
constexpr uint64_t lsbs = 0x0101010101010101ULL;
|
773
861
|
auto x = ctrl ^ (lsbs * hash);
|
774
|
-
return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x &
|
862
|
+
return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & kMsbs8Bytes);
|
775
863
|
}
|
776
864
|
|
777
865
|
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const {
|
778
|
-
constexpr uint64_t msbs = 0x8080808080808080ULL;
|
779
866
|
return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 6)) &
|
780
|
-
|
867
|
+
kMsbs8Bytes);
|
781
868
|
}
|
782
869
|
|
783
870
|
// Returns a bitmask representing the positions of full slots.
|
784
871
|
// Note: for `is_small()` tables group may contain the "same" slot twice:
|
785
872
|
// original and mirrored.
|
786
873
|
BitMask<uint64_t, kWidth, 3> MaskFull() const {
|
787
|
-
|
788
|
-
|
874
|
+
return BitMask<uint64_t, kWidth, 3>((ctrl ^ kMsbs8Bytes) & kMsbs8Bytes);
|
875
|
+
}
|
876
|
+
|
877
|
+
// Returns a bitmask representing the positions of non full slots.
|
878
|
+
// Note: this includes: kEmpty, kDeleted, kSentinel.
|
879
|
+
// It is useful in contexts when kSentinel is not present.
|
880
|
+
auto MaskNonFull() const {
|
881
|
+
return BitMask<uint64_t, kWidth, 3>(ctrl & kMsbs8Bytes);
|
789
882
|
}
|
790
883
|
|
791
884
|
NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const {
|
792
|
-
constexpr uint64_t msbs = 0x8080808080808080ULL;
|
793
885
|
return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & ~(ctrl << 7)) &
|
794
|
-
|
886
|
+
kMsbs8Bytes);
|
795
887
|
}
|
796
888
|
|
797
889
|
uint32_t CountLeadingEmptyOrDeleted() const {
|
@@ -803,9 +895,8 @@ struct GroupPortableImpl {
|
|
803
895
|
}
|
804
896
|
|
805
897
|
void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
|
806
|
-
constexpr uint64_t msbs = 0x8080808080808080ULL;
|
807
898
|
constexpr uint64_t lsbs = 0x0101010101010101ULL;
|
808
|
-
auto x = ctrl &
|
899
|
+
auto x = ctrl & kMsbs8Bytes;
|
809
900
|
auto res = (~x + (x >> 7)) & ~lsbs;
|
810
901
|
little_endian::Store64(dst, res);
|
811
902
|
}
|
@@ -815,21 +906,21 @@ struct GroupPortableImpl {
|
|
815
906
|
|
816
907
|
#ifdef ABSL_INTERNAL_HAVE_SSE2
|
817
908
|
using Group = GroupSse2Impl;
|
818
|
-
using
|
909
|
+
using GroupFullEmptyOrDeleted = GroupSse2Impl;
|
819
910
|
#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN)
|
820
911
|
using Group = GroupAArch64Impl;
|
821
912
|
// For Aarch64, we use the portable implementation for counting and masking
|
822
|
-
// empty or deleted group elements. This is to avoid the latency of moving
|
913
|
+
// full, empty or deleted group elements. This is to avoid the latency of moving
|
823
914
|
// between data GPRs and Neon registers when it does not provide a benefit.
|
824
915
|
// Using Neon is profitable when we call Match(), but is not when we don't,
|
825
|
-
// which is the case when we do *EmptyOrDeleted operations.
|
826
|
-
// make a similar approach beneficial on other architectures
|
827
|
-
// they have much lower GPR <-> vector register transfer
|
828
|
-
// Groups.
|
829
|
-
using
|
916
|
+
// which is the case when we do *EmptyOrDeleted and MaskFull operations.
|
917
|
+
// It is difficult to make a similar approach beneficial on other architectures
|
918
|
+
// such as x86 since they have much lower GPR <-> vector register transfer
|
919
|
+
// latency and 16-wide Groups.
|
920
|
+
using GroupFullEmptyOrDeleted = GroupPortableImpl;
|
830
921
|
#else
|
831
922
|
using Group = GroupPortableImpl;
|
832
|
-
using
|
923
|
+
using GroupFullEmptyOrDeleted = GroupPortableImpl;
|
833
924
|
#endif
|
834
925
|
|
835
926
|
// When there is an insertion with no reserved growth, we rehash with
|
@@ -978,17 +1069,96 @@ using CommonFieldsGenerationInfo = CommonFieldsGenerationInfoDisabled;
|
|
978
1069
|
using HashSetIteratorGenerationInfo = HashSetIteratorGenerationInfoDisabled;
|
979
1070
|
#endif
|
980
1071
|
|
1072
|
+
// Stored the information regarding number of slots we can still fill
|
1073
|
+
// without needing to rehash.
|
1074
|
+
//
|
1075
|
+
// We want to ensure sufficient number of empty slots in the table in order
|
1076
|
+
// to keep probe sequences relatively short. Empty slot in the probe group
|
1077
|
+
// is required to stop probing.
|
1078
|
+
//
|
1079
|
+
// Tombstones (kDeleted slots) are not included in the growth capacity,
|
1080
|
+
// because we'd like to rehash when the table is filled with tombstones and/or
|
1081
|
+
// full slots.
|
1082
|
+
//
|
1083
|
+
// GrowthInfo also stores a bit that encodes whether table may have any
|
1084
|
+
// deleted slots.
|
1085
|
+
// Most of the tables (>95%) have no deleted slots, so some functions can
|
1086
|
+
// be more efficient with this information.
|
1087
|
+
//
|
1088
|
+
// Callers can also force a rehash via the standard `rehash(0)`,
|
1089
|
+
// which will recompute this value as a side-effect.
|
1090
|
+
//
|
1091
|
+
// See also `CapacityToGrowth()`.
|
1092
|
+
class GrowthInfo {
|
1093
|
+
public:
|
1094
|
+
// Leaves data member uninitialized.
|
1095
|
+
GrowthInfo() = default;
|
1096
|
+
|
1097
|
+
// Initializes the GrowthInfo assuming we can grow `growth_left` elements
|
1098
|
+
// and there are no kDeleted slots in the table.
|
1099
|
+
void InitGrowthLeftNoDeleted(size_t growth_left) {
|
1100
|
+
growth_left_info_ = growth_left;
|
1101
|
+
}
|
1102
|
+
|
1103
|
+
// Overwrites single full slot with an empty slot.
|
1104
|
+
void OverwriteFullAsEmpty() { ++growth_left_info_; }
|
1105
|
+
|
1106
|
+
// Overwrites single empty slot with a full slot.
|
1107
|
+
void OverwriteEmptyAsFull() {
|
1108
|
+
assert(GetGrowthLeft() > 0);
|
1109
|
+
--growth_left_info_;
|
1110
|
+
}
|
1111
|
+
|
1112
|
+
// Overwrites several empty slots with full slots.
|
1113
|
+
void OverwriteManyEmptyAsFull(size_t cnt) {
|
1114
|
+
assert(GetGrowthLeft() >= cnt);
|
1115
|
+
growth_left_info_ -= cnt;
|
1116
|
+
}
|
1117
|
+
|
1118
|
+
// Overwrites specified control element with full slot.
|
1119
|
+
void OverwriteControlAsFull(ctrl_t ctrl) {
|
1120
|
+
assert(GetGrowthLeft() >= static_cast<size_t>(IsEmpty(ctrl)));
|
1121
|
+
growth_left_info_ -= static_cast<size_t>(IsEmpty(ctrl));
|
1122
|
+
}
|
1123
|
+
|
1124
|
+
// Overwrites single full slot with a deleted slot.
|
1125
|
+
void OverwriteFullAsDeleted() { growth_left_info_ |= kDeletedBit; }
|
1126
|
+
|
1127
|
+
// Returns true if table satisfies two properties:
|
1128
|
+
// 1. Guaranteed to have no kDeleted slots.
|
1129
|
+
// 2. There is a place for at least one element to grow.
|
1130
|
+
bool HasNoDeletedAndGrowthLeft() const {
|
1131
|
+
return static_cast<std::make_signed_t<size_t>>(growth_left_info_) > 0;
|
1132
|
+
}
|
1133
|
+
|
1134
|
+
// Returns true if the table satisfies two properties:
|
1135
|
+
// 1. Guaranteed to have no kDeleted slots.
|
1136
|
+
// 2. There is no growth left.
|
1137
|
+
bool HasNoGrowthLeftAndNoDeleted() const { return growth_left_info_ == 0; }
|
1138
|
+
|
1139
|
+
// Returns true if table guaranteed to have no k
|
1140
|
+
bool HasNoDeleted() const {
|
1141
|
+
return static_cast<std::make_signed_t<size_t>>(growth_left_info_) >= 0;
|
1142
|
+
}
|
1143
|
+
|
1144
|
+
// Returns the number of elements left to grow.
|
1145
|
+
size_t GetGrowthLeft() const { return growth_left_info_ & kGrowthLeftMask; }
|
1146
|
+
|
1147
|
+
private:
|
1148
|
+
static constexpr size_t kGrowthLeftMask = ((~size_t{}) >> 1);
|
1149
|
+
static constexpr size_t kDeletedBit = ~kGrowthLeftMask;
|
1150
|
+
// Topmost bit signal whenever there are deleted slots.
|
1151
|
+
size_t growth_left_info_;
|
1152
|
+
};
|
1153
|
+
|
1154
|
+
static_assert(sizeof(GrowthInfo) == sizeof(size_t), "");
|
1155
|
+
static_assert(alignof(GrowthInfo) == alignof(size_t), "");
|
1156
|
+
|
981
1157
|
// Returns whether `n` is a valid capacity (i.e., number of slots).
|
982
1158
|
//
|
983
1159
|
// A valid capacity is a non-zero integer `2^m - 1`.
|
984
1160
|
inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; }
|
985
1161
|
|
986
|
-
// Computes the offset from the start of the backing allocation of control.
|
987
|
-
// infoz and growth_left are stored at the beginning of the backing array.
|
988
|
-
inline size_t ControlOffset(bool has_infoz) {
|
989
|
-
return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(size_t);
|
990
|
-
}
|
991
|
-
|
992
1162
|
// Returns the number of "cloned control bytes".
|
993
1163
|
//
|
994
1164
|
// This is the number of control bytes that are present both at the beginning
|
@@ -996,36 +1166,157 @@ inline size_t ControlOffset(bool has_infoz) {
|
|
996
1166
|
// `Group::kWidth`-width probe window starting from any control byte.
|
997
1167
|
constexpr size_t NumClonedBytes() { return Group::kWidth - 1; }
|
998
1168
|
|
999
|
-
//
|
1000
|
-
|
1001
|
-
|
1002
|
-
assert(IsValidCapacity(capacity));
|
1003
|
-
const size_t num_control_bytes = capacity + 1 + NumClonedBytes();
|
1004
|
-
return ControlOffset(has_infoz) + num_control_bytes;
|
1169
|
+
// Returns the number of control bytes including cloned.
|
1170
|
+
constexpr size_t NumControlBytes(size_t capacity) {
|
1171
|
+
return capacity + 1 + NumClonedBytes();
|
1005
1172
|
}
|
1006
1173
|
|
1007
|
-
//
|
1008
|
-
//
|
1009
|
-
inline size_t
|
1010
|
-
|
1011
|
-
return (GenerationOffset(capacity, has_infoz) + NumGenerationBytes() +
|
1012
|
-
slot_align - 1) &
|
1013
|
-
(~slot_align + 1);
|
1174
|
+
// Computes the offset from the start of the backing allocation of control.
|
1175
|
+
// infoz and growth_info are stored at the beginning of the backing array.
|
1176
|
+
inline static size_t ControlOffset(bool has_infoz) {
|
1177
|
+
return (has_infoz ? sizeof(HashtablezInfoHandle) : 0) + sizeof(GrowthInfo);
|
1014
1178
|
}
|
1015
1179
|
|
1016
|
-
//
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1180
|
+
// Helper class for computing offsets and allocation size of hash set fields.
|
1181
|
+
class RawHashSetLayout {
|
1182
|
+
public:
|
1183
|
+
explicit RawHashSetLayout(size_t capacity, size_t slot_align, bool has_infoz)
|
1184
|
+
: capacity_(capacity),
|
1185
|
+
control_offset_(ControlOffset(has_infoz)),
|
1186
|
+
generation_offset_(control_offset_ + NumControlBytes(capacity)),
|
1187
|
+
slot_offset_(
|
1188
|
+
(generation_offset_ + NumGenerationBytes() + slot_align - 1) &
|
1189
|
+
(~slot_align + 1)) {
|
1190
|
+
assert(IsValidCapacity(capacity));
|
1191
|
+
}
|
1192
|
+
|
1193
|
+
// Returns the capacity of a table.
|
1194
|
+
size_t capacity() const { return capacity_; }
|
1195
|
+
|
1196
|
+
// Returns precomputed offset from the start of the backing allocation of
|
1197
|
+
// control.
|
1198
|
+
size_t control_offset() const { return control_offset_; }
|
1199
|
+
|
1200
|
+
// Given the capacity of a table, computes the offset (from the start of the
|
1201
|
+
// backing allocation) of the generation counter (if it exists).
|
1202
|
+
size_t generation_offset() const { return generation_offset_; }
|
1203
|
+
|
1204
|
+
// Given the capacity of a table, computes the offset (from the start of the
|
1205
|
+
// backing allocation) at which the slots begin.
|
1206
|
+
size_t slot_offset() const { return slot_offset_; }
|
1207
|
+
|
1208
|
+
// Given the capacity of a table, computes the total size of the backing
|
1209
|
+
// array.
|
1210
|
+
size_t alloc_size(size_t slot_size) const {
|
1211
|
+
return slot_offset_ + capacity_ * slot_size;
|
1212
|
+
}
|
1213
|
+
|
1214
|
+
private:
|
1215
|
+
size_t capacity_;
|
1216
|
+
size_t control_offset_;
|
1217
|
+
size_t generation_offset_;
|
1218
|
+
size_t slot_offset_;
|
1219
|
+
};
|
1220
|
+
|
1221
|
+
struct HashtableFreeFunctionsAccess;
|
1222
|
+
|
1223
|
+
// We only allow a maximum of 1 SOO element, which makes the implementation
|
1224
|
+
// much simpler. Complications with multiple SOO elements include:
|
1225
|
+
// - Satisfying the guarantee that erasing one element doesn't invalidate
|
1226
|
+
// iterators to other elements means we would probably need actual SOO
|
1227
|
+
// control bytes.
|
1228
|
+
// - In order to prevent user code from depending on iteration order for small
|
1229
|
+
// tables, we would need to randomize the iteration order somehow.
|
1230
|
+
constexpr size_t SooCapacity() { return 1; }
|
1231
|
+
// Sentinel type to indicate SOO CommonFields construction.
|
1232
|
+
struct soo_tag_t {};
|
1233
|
+
// Sentinel type to indicate SOO CommonFields construction with full size.
|
1234
|
+
struct full_soo_tag_t {};
|
1235
|
+
|
1236
|
+
// Suppress erroneous uninitialized memory errors on GCC. For example, GCC
|
1237
|
+
// thinks that the call to slot_array() in find_or_prepare_insert() is reading
|
1238
|
+
// uninitialized memory, but slot_array is only called there when the table is
|
1239
|
+
// non-empty and this memory is initialized when the table is non-empty.
|
1240
|
+
#if !defined(__clang__) && defined(__GNUC__)
|
1241
|
+
#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) \
|
1242
|
+
_Pragma("GCC diagnostic push") \
|
1243
|
+
_Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") \
|
1244
|
+
_Pragma("GCC diagnostic ignored \"-Wuninitialized\"") x; \
|
1245
|
+
_Pragma("GCC diagnostic pop")
|
1246
|
+
#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) \
|
1247
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(return x)
|
1248
|
+
#else
|
1249
|
+
#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) x
|
1250
|
+
#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) return x
|
1251
|
+
#endif
|
1252
|
+
|
1253
|
+
// This allows us to work around an uninitialized memory warning when
|
1254
|
+
// constructing begin() iterators in empty hashtables.
|
1255
|
+
union MaybeInitializedPtr {
|
1256
|
+
void* get() const { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(p); }
|
1257
|
+
void set(void* ptr) { p = ptr; }
|
1258
|
+
|
1259
|
+
void* p;
|
1260
|
+
};
|
1261
|
+
|
1262
|
+
struct HeapPtrs {
|
1263
|
+
HeapPtrs() = default;
|
1264
|
+
explicit HeapPtrs(ctrl_t* c) : control(c) {}
|
1265
|
+
|
1266
|
+
// The control bytes (and, also, a pointer near to the base of the backing
|
1267
|
+
// array).
|
1268
|
+
//
|
1269
|
+
// This contains `capacity + 1 + NumClonedBytes()` entries, even
|
1270
|
+
// when the table is empty (hence EmptyGroup).
|
1271
|
+
//
|
1272
|
+
// Note that growth_info is stored immediately before this pointer.
|
1273
|
+
// May be uninitialized for SOO tables.
|
1274
|
+
ctrl_t* control;
|
1275
|
+
|
1276
|
+
// The beginning of the slots, located at `SlotOffset()` bytes after
|
1277
|
+
// `control`. May be uninitialized for empty tables.
|
1278
|
+
// Note: we can't use `slots` because Qt defines "slots" as a macro.
|
1279
|
+
MaybeInitializedPtr slot_array;
|
1280
|
+
};
|
1281
|
+
|
1282
|
+
// Manages the backing array pointers or the SOO slot. When raw_hash_set::is_soo
|
1283
|
+
// is true, the SOO slot is stored in `soo_data`. Otherwise, we use `heap`.
|
1284
|
+
union HeapOrSoo {
|
1285
|
+
HeapOrSoo() = default;
|
1286
|
+
explicit HeapOrSoo(ctrl_t* c) : heap(c) {}
|
1287
|
+
|
1288
|
+
ctrl_t*& control() {
|
1289
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control);
|
1290
|
+
}
|
1291
|
+
ctrl_t* control() const {
|
1292
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control);
|
1293
|
+
}
|
1294
|
+
MaybeInitializedPtr& slot_array() {
|
1295
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array);
|
1296
|
+
}
|
1297
|
+
MaybeInitializedPtr slot_array() const {
|
1298
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array);
|
1299
|
+
}
|
1300
|
+
void* get_soo_data() {
|
1301
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data);
|
1302
|
+
}
|
1303
|
+
const void* get_soo_data() const {
|
1304
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data);
|
1305
|
+
}
|
1306
|
+
|
1307
|
+
HeapPtrs heap;
|
1308
|
+
unsigned char soo_data[sizeof(HeapPtrs)];
|
1309
|
+
};
|
1022
1310
|
|
1023
1311
|
// CommonFields hold the fields in raw_hash_set that do not depend
|
1024
1312
|
// on template parameters. This allows us to conveniently pass all
|
1025
1313
|
// of this state to helper functions as a single argument.
|
1026
1314
|
class CommonFields : public CommonFieldsGenerationInfo {
|
1027
1315
|
public:
|
1028
|
-
CommonFields()
|
1316
|
+
CommonFields() : capacity_(0), size_(0), heap_or_soo_(EmptyGroup()) {}
|
1317
|
+
explicit CommonFields(soo_tag_t) : capacity_(SooCapacity()), size_(0) {}
|
1318
|
+
explicit CommonFields(full_soo_tag_t)
|
1319
|
+
: capacity_(SooCapacity()), size_(size_t{1} << HasInfozShift()) {}
|
1029
1320
|
|
1030
1321
|
// Not copyable
|
1031
1322
|
CommonFields(const CommonFields&) = delete;
|
@@ -1035,23 +1326,44 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1035
1326
|
CommonFields(CommonFields&& that) = default;
|
1036
1327
|
CommonFields& operator=(CommonFields&&) = default;
|
1037
1328
|
|
1038
|
-
|
1039
|
-
|
1329
|
+
template <bool kSooEnabled>
|
1330
|
+
static CommonFields CreateDefault() {
|
1331
|
+
return kSooEnabled ? CommonFields{soo_tag_t{}} : CommonFields{};
|
1332
|
+
}
|
1333
|
+
|
1334
|
+
// The inline data for SOO is written on top of control_/slots_.
|
1335
|
+
const void* soo_data() const { return heap_or_soo_.get_soo_data(); }
|
1336
|
+
void* soo_data() { return heap_or_soo_.get_soo_data(); }
|
1337
|
+
|
1338
|
+
HeapOrSoo heap_or_soo() const { return heap_or_soo_; }
|
1339
|
+
const HeapOrSoo& heap_or_soo_ref() const { return heap_or_soo_; }
|
1340
|
+
|
1341
|
+
ctrl_t* control() const { return heap_or_soo_.control(); }
|
1342
|
+
void set_control(ctrl_t* c) { heap_or_soo_.control() = c; }
|
1040
1343
|
void* backing_array_start() const {
|
1041
|
-
//
|
1344
|
+
// growth_info (and maybe infoz) is stored before control bytes.
|
1042
1345
|
assert(reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0);
|
1043
1346
|
return control() - ControlOffset(has_infoz());
|
1044
1347
|
}
|
1045
1348
|
|
1046
1349
|
// Note: we can't use slots() because Qt defines "slots" as a macro.
|
1047
|
-
void* slot_array() const { return
|
1048
|
-
|
1350
|
+
void* slot_array() const { return heap_or_soo_.slot_array().get(); }
|
1351
|
+
MaybeInitializedPtr slots_union() const { return heap_or_soo_.slot_array(); }
|
1352
|
+
void set_slots(void* s) { heap_or_soo_.slot_array().set(s); }
|
1049
1353
|
|
1050
1354
|
// The number of filled slots.
|
1051
1355
|
size_t size() const { return size_ >> HasInfozShift(); }
|
1052
1356
|
void set_size(size_t s) {
|
1053
1357
|
size_ = (s << HasInfozShift()) | (size_ & HasInfozMask());
|
1054
1358
|
}
|
1359
|
+
void set_empty_soo() {
|
1360
|
+
AssertInSooMode();
|
1361
|
+
size_ = 0;
|
1362
|
+
}
|
1363
|
+
void set_full_soo() {
|
1364
|
+
AssertInSooMode();
|
1365
|
+
size_ = size_t{1} << HasInfozShift();
|
1366
|
+
}
|
1055
1367
|
void increment_size() {
|
1056
1368
|
assert(size() < capacity());
|
1057
1369
|
size_ += size_t{1} << HasInfozShift();
|
@@ -1070,15 +1382,17 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1070
1382
|
|
1071
1383
|
// The number of slots we can still fill without needing to rehash.
|
1072
1384
|
// This is stored in the heap allocation before the control bytes.
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1385
|
+
// TODO(b/289225379): experiment with moving growth_info back inline to
|
1386
|
+
// increase room for SOO.
|
1387
|
+
size_t growth_left() const { return growth_info().GetGrowthLeft(); }
|
1388
|
+
|
1389
|
+
GrowthInfo& growth_info() {
|
1390
|
+
auto* gl_ptr = reinterpret_cast<GrowthInfo*>(control()) - 1;
|
1391
|
+
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(GrowthInfo) == 0);
|
1076
1392
|
return *gl_ptr;
|
1077
1393
|
}
|
1078
|
-
|
1079
|
-
|
1080
|
-
assert(reinterpret_cast<uintptr_t>(gl_ptr) % alignof(size_t) == 0);
|
1081
|
-
*gl_ptr = gl;
|
1394
|
+
GrowthInfo growth_info() const {
|
1395
|
+
return const_cast<CommonFields*>(this)->growth_info();
|
1082
1396
|
}
|
1083
1397
|
|
1084
1398
|
bool has_infoz() const {
|
@@ -1103,12 +1417,8 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1103
1417
|
should_rehash_for_bug_detection_on_insert(control(), capacity());
|
1104
1418
|
}
|
1105
1419
|
bool should_rehash_for_bug_detection_on_move() const {
|
1106
|
-
return CommonFieldsGenerationInfo::
|
1107
|
-
|
1108
|
-
}
|
1109
|
-
void maybe_increment_generation_on_move() {
|
1110
|
-
if (capacity() == 0) return;
|
1111
|
-
increment_generation();
|
1420
|
+
return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move(
|
1421
|
+
control(), capacity());
|
1112
1422
|
}
|
1113
1423
|
void reset_reserved_growth(size_t reservation) {
|
1114
1424
|
CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size());
|
@@ -1116,7 +1426,16 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1116
1426
|
|
1117
1427
|
// The size of the backing array allocation.
|
1118
1428
|
size_t alloc_size(size_t slot_size, size_t slot_align) const {
|
1119
|
-
return
|
1429
|
+
return RawHashSetLayout(capacity(), slot_align, has_infoz())
|
1430
|
+
.alloc_size(slot_size);
|
1431
|
+
}
|
1432
|
+
|
1433
|
+
// Move fields other than heap_or_soo_.
|
1434
|
+
void move_non_heap_or_soo_fields(CommonFields& that) {
|
1435
|
+
static_cast<CommonFieldsGenerationInfo&>(*this) =
|
1436
|
+
std::move(static_cast<CommonFieldsGenerationInfo&>(that));
|
1437
|
+
capacity_ = that.capacity_;
|
1438
|
+
size_ = that.size_;
|
1120
1439
|
}
|
1121
1440
|
|
1122
1441
|
// Returns the number of control bytes set to kDeleted. For testing only.
|
@@ -1132,21 +1451,12 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1132
1451
|
return (size_t{1} << HasInfozShift()) - 1;
|
1133
1452
|
}
|
1134
1453
|
|
1135
|
-
//
|
1136
|
-
//
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
// This contains `capacity + 1 + NumClonedBytes()` entries, even
|
1142
|
-
// when the table is empty (hence EmptyGroup).
|
1143
|
-
//
|
1144
|
-
// Note that growth_left is stored immediately before this pointer.
|
1145
|
-
ctrl_t* control_ = EmptyGroup();
|
1146
|
-
|
1147
|
-
// The beginning of the slots, located at `SlotOffset()` bytes after
|
1148
|
-
// `control`. May be null for empty tables.
|
1149
|
-
void* slots_ = nullptr;
|
1454
|
+
// We can't assert that SOO is enabled because we don't have SooEnabled(), but
|
1455
|
+
// we assert what we can.
|
1456
|
+
void AssertInSooMode() const {
|
1457
|
+
assert(capacity() == SooCapacity());
|
1458
|
+
assert(!has_infoz());
|
1459
|
+
}
|
1150
1460
|
|
1151
1461
|
// The number of slots in the backing array. This is always 2^N-1 for an
|
1152
1462
|
// integer N. NOTE: we tried experimenting with compressing the capacity and
|
@@ -1154,10 +1464,16 @@ class CommonFields : public CommonFieldsGenerationInfo {
|
|
1154
1464
|
// power (N in 2^N-1), and (b) storing 2^N as the most significant bit of
|
1155
1465
|
// size_ and storing size in the low bits. Both of these experiments were
|
1156
1466
|
// regressions, presumably because we need capacity to do find operations.
|
1157
|
-
size_t capacity_
|
1467
|
+
size_t capacity_;
|
1158
1468
|
|
1159
1469
|
// The size and also has one bit that stores whether we have infoz.
|
1160
|
-
|
1470
|
+
// TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_
|
1471
|
+
// encode the size in SOO case. We would be making size()/capacity() more
|
1472
|
+
// expensive in order to have more SOO space.
|
1473
|
+
size_t size_;
|
1474
|
+
|
1475
|
+
// Either the control/slots pointers or the SOO slot.
|
1476
|
+
HeapOrSoo heap_or_soo_;
|
1161
1477
|
};
|
1162
1478
|
|
1163
1479
|
template <class Policy, class Hash, class Eq, class Alloc>
|
@@ -1320,6 +1636,10 @@ inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a,
|
|
1320
1636
|
const void* const& slot_b) {
|
1321
1637
|
// If either control byte is null, then we can't tell.
|
1322
1638
|
if (ctrl_a == nullptr || ctrl_b == nullptr) return true;
|
1639
|
+
const bool a_is_soo = IsSooControl(ctrl_a);
|
1640
|
+
if (a_is_soo != IsSooControl(ctrl_b)) return false;
|
1641
|
+
if (a_is_soo) return slot_a == slot_b;
|
1642
|
+
|
1323
1643
|
const void* low_slot = slot_a;
|
1324
1644
|
const void* hi_slot = slot_b;
|
1325
1645
|
if (ctrl_a > ctrl_b) {
|
@@ -1343,41 +1663,45 @@ inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b,
|
|
1343
1663
|
// - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout
|
1344
1664
|
// - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve
|
1345
1665
|
// the chances that the hot paths will be inlined.
|
1666
|
+
|
1667
|
+
// fail_if(is_invalid, message) crashes when is_invalid is true and provides
|
1668
|
+
// an error message based on `message`.
|
1669
|
+
const auto fail_if = [](bool is_invalid, const char* message) {
|
1670
|
+
if (ABSL_PREDICT_FALSE(is_invalid)) {
|
1671
|
+
ABSL_RAW_LOG(FATAL, "Invalid iterator comparison. %s", message);
|
1672
|
+
}
|
1673
|
+
};
|
1674
|
+
|
1346
1675
|
const bool a_is_default = ctrl_a == EmptyGroup();
|
1347
1676
|
const bool b_is_default = ctrl_b == EmptyGroup();
|
1348
|
-
if (ABSL_PREDICT_FALSE(a_is_default != b_is_default)) {
|
1349
|
-
ABSL_RAW_LOG(
|
1350
|
-
FATAL,
|
1351
|
-
"Invalid iterator comparison. Comparing default-constructed iterator "
|
1352
|
-
"with non-default-constructed iterator.");
|
1353
|
-
}
|
1354
1677
|
if (a_is_default && b_is_default) return;
|
1678
|
+
fail_if(a_is_default != b_is_default,
|
1679
|
+
"Comparing default-constructed hashtable iterator with a "
|
1680
|
+
"non-default-constructed hashtable iterator.");
|
1355
1681
|
|
1356
1682
|
if (SwisstableGenerationsEnabled()) {
|
1357
1683
|
if (ABSL_PREDICT_TRUE(generation_ptr_a == generation_ptr_b)) return;
|
1684
|
+
// Users don't need to know whether the tables are SOO so don't mention SOO
|
1685
|
+
// in the debug message.
|
1686
|
+
const bool a_is_soo = IsSooControl(ctrl_a);
|
1687
|
+
const bool b_is_soo = IsSooControl(ctrl_b);
|
1688
|
+
fail_if(a_is_soo != b_is_soo || (a_is_soo && b_is_soo),
|
1689
|
+
"Comparing iterators from different hashtables.");
|
1690
|
+
|
1358
1691
|
const bool a_is_empty = IsEmptyGeneration(generation_ptr_a);
|
1359
1692
|
const bool b_is_empty = IsEmptyGeneration(generation_ptr_b);
|
1360
|
-
|
1361
|
-
|
1362
|
-
|
1363
|
-
|
1364
|
-
|
1365
|
-
|
1366
|
-
if (a_is_empty && b_is_empty) {
|
1367
|
-
ABSL_RAW_LOG(FATAL,
|
1368
|
-
"Invalid iterator comparison. Comparing iterators from "
|
1369
|
-
"different empty hashtables.");
|
1370
|
-
}
|
1693
|
+
fail_if(a_is_empty != b_is_empty,
|
1694
|
+
"Comparing an iterator from an empty hashtable with an iterator "
|
1695
|
+
"from a non-empty hashtable.");
|
1696
|
+
fail_if(a_is_empty && b_is_empty,
|
1697
|
+
"Comparing iterators from different empty hashtables.");
|
1698
|
+
|
1371
1699
|
const bool a_is_end = ctrl_a == nullptr;
|
1372
1700
|
const bool b_is_end = ctrl_b == nullptr;
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1376
|
-
|
1377
|
-
}
|
1378
|
-
ABSL_RAW_LOG(FATAL,
|
1379
|
-
"Invalid iterator comparison. Comparing non-end() iterators "
|
1380
|
-
"from different hashtables.");
|
1701
|
+
fail_if(a_is_end || b_is_end,
|
1702
|
+
"Comparing iterator with an end() iterator from a different "
|
1703
|
+
"hashtable.");
|
1704
|
+
fail_if(true, "Comparing non-end() iterators from different hashtables.");
|
1381
1705
|
} else {
|
1382
1706
|
ABSL_HARDENING_ASSERT(
|
1383
1707
|
AreItersFromSameContainer(ctrl_a, ctrl_b, slot_a, slot_b) &&
|
@@ -1432,20 +1756,17 @@ template <typename = void>
|
|
1432
1756
|
inline FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
|
1433
1757
|
auto seq = probe(common, hash);
|
1434
1758
|
const ctrl_t* ctrl = common.control();
|
1759
|
+
if (IsEmptyOrDeleted(ctrl[seq.offset()]) &&
|
1760
|
+
!ShouldInsertBackwards(common.capacity(), hash, ctrl)) {
|
1761
|
+
return {seq.offset(), /*probe_length=*/0};
|
1762
|
+
}
|
1435
1763
|
while (true) {
|
1436
|
-
|
1764
|
+
GroupFullEmptyOrDeleted g{ctrl + seq.offset()};
|
1437
1765
|
auto mask = g.MaskEmptyOrDeleted();
|
1438
1766
|
if (mask) {
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1442
|
-
// the group.
|
1443
|
-
// TODO(kfm,sbenza): revisit after we do unconditional mixing
|
1444
|
-
if (!is_small(common.capacity()) && ShouldInsertBackwards(hash, ctrl)) {
|
1445
|
-
return {seq.offset(mask.HighestBitSet()), seq.index()};
|
1446
|
-
}
|
1447
|
-
#endif
|
1448
|
-
return {seq.offset(mask.LowestBitSet()), seq.index()};
|
1767
|
+
return {
|
1768
|
+
seq.offset(GetInsertionOffset(mask, common.capacity(), hash, ctrl)),
|
1769
|
+
seq.index()};
|
1449
1770
|
}
|
1450
1771
|
seq.next();
|
1451
1772
|
assert(seq.index() <= common.capacity() && "full table!");
|
@@ -1462,7 +1783,8 @@ extern template FindInfo find_first_non_full(const CommonFields&, size_t);
|
|
1462
1783
|
FindInfo find_first_non_full_outofline(const CommonFields&, size_t);
|
1463
1784
|
|
1464
1785
|
inline void ResetGrowthLeft(CommonFields& common) {
|
1465
|
-
common.
|
1786
|
+
common.growth_info().InitGrowthLeftNoDeleted(
|
1787
|
+
CapacityToGrowth(common.capacity()) - common.size());
|
1466
1788
|
}
|
1467
1789
|
|
1468
1790
|
// Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire
|
@@ -1476,43 +1798,140 @@ inline void ResetCtrl(CommonFields& common, size_t slot_size) {
|
|
1476
1798
|
SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity);
|
1477
1799
|
}
|
1478
1800
|
|
1479
|
-
// Sets
|
1480
|
-
|
1481
|
-
|
1482
|
-
|
1483
|
-
|
1484
|
-
size_t slot_size) {
|
1485
|
-
const size_t capacity = common.capacity();
|
1486
|
-
assert(i < capacity);
|
1487
|
-
|
1488
|
-
auto* slot_i = static_cast<const char*>(common.slot_array()) + i * slot_size;
|
1801
|
+
// Sets sanitizer poisoning for slot corresponding to control byte being set.
|
1802
|
+
inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h,
|
1803
|
+
size_t slot_size) {
|
1804
|
+
assert(i < c.capacity());
|
1805
|
+
auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size;
|
1489
1806
|
if (IsFull(h)) {
|
1490
1807
|
SanitizerUnpoisonMemoryRegion(slot_i, slot_size);
|
1491
1808
|
} else {
|
1492
1809
|
SanitizerPoisonMemoryRegion(slot_i, slot_size);
|
1493
1810
|
}
|
1811
|
+
}
|
1494
1812
|
|
1495
|
-
|
1813
|
+
// Sets `ctrl[i]` to `h`.
|
1814
|
+
//
|
1815
|
+
// Unlike setting it directly, this function will perform bounds checks and
|
1816
|
+
// mirror the value to the cloned tail if necessary.
|
1817
|
+
inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h,
|
1818
|
+
size_t slot_size) {
|
1819
|
+
DoSanitizeOnSetCtrl(c, i, h, slot_size);
|
1820
|
+
ctrl_t* ctrl = c.control();
|
1496
1821
|
ctrl[i] = h;
|
1497
|
-
ctrl[((i - NumClonedBytes()) & capacity
|
1822
|
+
ctrl[((i - NumClonedBytes()) & c.capacity()) +
|
1823
|
+
(NumClonedBytes() & c.capacity())] = h;
|
1824
|
+
}
|
1825
|
+
// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
|
1826
|
+
inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) {
|
1827
|
+
SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size);
|
1498
1828
|
}
|
1499
1829
|
|
1830
|
+
// Like SetCtrl, but in a single group table, we can save some operations when
|
1831
|
+
// setting the cloned control byte.
|
1832
|
+
inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h,
|
1833
|
+
size_t slot_size) {
|
1834
|
+
assert(is_single_group(c.capacity()));
|
1835
|
+
DoSanitizeOnSetCtrl(c, i, h, slot_size);
|
1836
|
+
ctrl_t* ctrl = c.control();
|
1837
|
+
ctrl[i] = h;
|
1838
|
+
ctrl[i + c.capacity() + 1] = h;
|
1839
|
+
}
|
1500
1840
|
// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
|
1501
|
-
inline void
|
1502
|
-
|
1503
|
-
|
1841
|
+
inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h,
|
1842
|
+
size_t slot_size) {
|
1843
|
+
SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size);
|
1504
1844
|
}
|
1505
1845
|
|
1506
|
-
//
|
1846
|
+
// growth_info (which is a size_t) is stored with the backing array.
|
1507
1847
|
constexpr size_t BackingArrayAlignment(size_t align_of_slot) {
|
1508
|
-
return (std::max)(align_of_slot, alignof(
|
1848
|
+
return (std::max)(align_of_slot, alignof(GrowthInfo));
|
1509
1849
|
}
|
1510
1850
|
|
1511
1851
|
// Returns the address of the ith slot in slots where each slot occupies
|
1512
1852
|
// slot_size.
|
1513
1853
|
inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
|
1514
|
-
return
|
1515
|
-
|
1854
|
+
return static_cast<void*>(static_cast<char*>(slot_array) +
|
1855
|
+
(slot * slot_size));
|
1856
|
+
}
|
1857
|
+
|
1858
|
+
// Iterates over all full slots and calls `cb(const ctrl_t*, SlotType*)`.
|
1859
|
+
// No insertion to the table allowed during Callback call.
|
1860
|
+
// Erasure is allowed only for the element passed to the callback.
|
1861
|
+
template <class SlotType, class Callback>
|
1862
|
+
ABSL_ATTRIBUTE_ALWAYS_INLINE inline void IterateOverFullSlots(
|
1863
|
+
const CommonFields& c, SlotType* slot, Callback cb) {
|
1864
|
+
const size_t cap = c.capacity();
|
1865
|
+
const ctrl_t* ctrl = c.control();
|
1866
|
+
if (is_small(cap)) {
|
1867
|
+
// Mirrored/cloned control bytes in small table are also located in the
|
1868
|
+
// first group (starting from position 0). We are taking group from position
|
1869
|
+
// `capacity` in order to avoid duplicates.
|
1870
|
+
|
1871
|
+
// Small tables capacity fits into portable group, where
|
1872
|
+
// GroupPortableImpl::MaskFull is more efficient for the
|
1873
|
+
// capacity <= GroupPortableImpl::kWidth.
|
1874
|
+
assert(cap <= GroupPortableImpl::kWidth &&
|
1875
|
+
"unexpectedly large small capacity");
|
1876
|
+
static_assert(Group::kWidth >= GroupPortableImpl::kWidth,
|
1877
|
+
"unexpected group width");
|
1878
|
+
// Group starts from kSentinel slot, so indices in the mask will
|
1879
|
+
// be increased by 1.
|
1880
|
+
const auto mask = GroupPortableImpl(ctrl + cap).MaskFull();
|
1881
|
+
--ctrl;
|
1882
|
+
--slot;
|
1883
|
+
for (uint32_t i : mask) {
|
1884
|
+
cb(ctrl + i, slot + i);
|
1885
|
+
}
|
1886
|
+
return;
|
1887
|
+
}
|
1888
|
+
size_t remaining = c.size();
|
1889
|
+
ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = remaining;
|
1890
|
+
while (remaining != 0) {
|
1891
|
+
for (uint32_t i : GroupFullEmptyOrDeleted(ctrl).MaskFull()) {
|
1892
|
+
assert(IsFull(ctrl[i]) && "hash table was modified unexpectedly");
|
1893
|
+
cb(ctrl + i, slot + i);
|
1894
|
+
--remaining;
|
1895
|
+
}
|
1896
|
+
ctrl += Group::kWidth;
|
1897
|
+
slot += Group::kWidth;
|
1898
|
+
assert((remaining == 0 || *(ctrl - 1) != ctrl_t::kSentinel) &&
|
1899
|
+
"hash table was modified unexpectedly");
|
1900
|
+
}
|
1901
|
+
// NOTE: erasure of the current element is allowed in callback for
|
1902
|
+
// absl::erase_if specialization. So we use `>=`.
|
1903
|
+
assert(original_size_for_assert >= c.size() &&
|
1904
|
+
"hash table was modified unexpectedly");
|
1905
|
+
}
|
1906
|
+
|
1907
|
+
template <typename CharAlloc>
|
1908
|
+
constexpr bool ShouldSampleHashtablezInfo() {
|
1909
|
+
// Folks with custom allocators often make unwarranted assumptions about the
|
1910
|
+
// behavior of their classes vis-a-vis trivial destructability and what
|
1911
|
+
// calls they will or won't make. Avoid sampling for people with custom
|
1912
|
+
// allocators to get us out of this mess. This is not a hard guarantee but
|
1913
|
+
// a workaround while we plan the exact guarantee we want to provide.
|
1914
|
+
return std::is_same<CharAlloc, std::allocator<char>>::value;
|
1915
|
+
}
|
1916
|
+
|
1917
|
+
template <bool kSooEnabled>
|
1918
|
+
HashtablezInfoHandle SampleHashtablezInfo(size_t sizeof_slot, size_t sizeof_key,
|
1919
|
+
size_t sizeof_value,
|
1920
|
+
size_t old_capacity, bool was_soo,
|
1921
|
+
HashtablezInfoHandle forced_infoz,
|
1922
|
+
CommonFields& c) {
|
1923
|
+
if (forced_infoz.IsSampled()) return forced_infoz;
|
1924
|
+
// In SOO, we sample on the first insertion so if this is an empty SOO case
|
1925
|
+
// (e.g. when reserve is called), then we still need to sample.
|
1926
|
+
if (kSooEnabled && was_soo && c.size() == 0) {
|
1927
|
+
return Sample(sizeof_slot, sizeof_key, sizeof_value, SooCapacity());
|
1928
|
+
}
|
1929
|
+
// For non-SOO cases, we sample whenever the capacity is increasing from zero
|
1930
|
+
// to non-zero.
|
1931
|
+
if (!kSooEnabled && old_capacity == 0) {
|
1932
|
+
return Sample(sizeof_slot, sizeof_key, sizeof_value, 0);
|
1933
|
+
}
|
1934
|
+
return c.infoz();
|
1516
1935
|
}
|
1517
1936
|
|
1518
1937
|
// Helper class to perform resize of the hash set.
|
@@ -1521,17 +1940,21 @@ inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
|
|
1521
1940
|
// See GrowIntoSingleGroupShuffleControlBytes for details.
|
1522
1941
|
class HashSetResizeHelper {
|
1523
1942
|
public:
|
1524
|
-
explicit HashSetResizeHelper(CommonFields& c
|
1525
|
-
|
1526
|
-
|
1527
|
-
had_infoz_(c.has_infoz())
|
1528
|
-
|
1529
|
-
|
1530
|
-
|
1943
|
+
explicit HashSetResizeHelper(CommonFields& c, bool was_soo, bool had_soo_slot,
|
1944
|
+
HashtablezInfoHandle forced_infoz)
|
1945
|
+
: old_capacity_(c.capacity()),
|
1946
|
+
had_infoz_(c.has_infoz()),
|
1947
|
+
was_soo_(was_soo),
|
1948
|
+
had_soo_slot_(had_soo_slot),
|
1949
|
+
forced_infoz_(forced_infoz) {}
|
1950
|
+
|
1951
|
+
// Optimized for small groups version of `find_first_non_full`.
|
1952
|
+
// Beneficial only right after calling `raw_hash_set::resize`.
|
1953
|
+
// It is safe to call in case capacity is big or was not changed, but there
|
1954
|
+
// will be no performance benefit.
|
1531
1955
|
// It has implicit assumption that `resize` will call
|
1532
1956
|
// `GrowSizeIntoSingleGroup*` in case `IsGrowingIntoSingleGroupApplicable`.
|
1533
|
-
// Falls back to `find_first_non_full` in case of big groups
|
1534
|
-
// safe to use after `rehash_and_grow_if_necessary`.
|
1957
|
+
// Falls back to `find_first_non_full` in case of big groups.
|
1535
1958
|
static FindInfo FindFirstNonFullAfterResize(const CommonFields& c,
|
1536
1959
|
size_t old_capacity,
|
1537
1960
|
size_t hash) {
|
@@ -1553,14 +1976,30 @@ class HashSetResizeHelper {
|
|
1553
1976
|
return FindInfo{offset, 0};
|
1554
1977
|
}
|
1555
1978
|
|
1556
|
-
|
1979
|
+
HeapOrSoo& old_heap_or_soo() { return old_heap_or_soo_; }
|
1980
|
+
void* old_soo_data() { return old_heap_or_soo_.get_soo_data(); }
|
1981
|
+
ctrl_t* old_ctrl() const {
|
1982
|
+
assert(!was_soo_);
|
1983
|
+
return old_heap_or_soo_.control();
|
1984
|
+
}
|
1985
|
+
void* old_slots() const {
|
1986
|
+
assert(!was_soo_);
|
1987
|
+
return old_heap_or_soo_.slot_array().get();
|
1988
|
+
}
|
1557
1989
|
size_t old_capacity() const { return old_capacity_; }
|
1558
1990
|
|
1991
|
+
// Returns the index of the SOO slot when growing from SOO to non-SOO in a
|
1992
|
+
// single group. See also InitControlBytesAfterSoo(). It's important to use
|
1993
|
+
// index 1 so that when resizing from capacity 1 to 3, we can still have
|
1994
|
+
// random iteration order between the first two inserted elements.
|
1995
|
+
// I.e. it allows inserting the second element at either index 0 or 2.
|
1996
|
+
static size_t SooSlotIndex() { return 1; }
|
1997
|
+
|
1559
1998
|
// Allocates a backing array for the hashtable.
|
1560
1999
|
// Reads `capacity` and updates all other fields based on the result of
|
1561
2000
|
// the allocation.
|
1562
2001
|
//
|
1563
|
-
// It also may do the
|
2002
|
+
// It also may do the following actions:
|
1564
2003
|
// 1. initialize control bytes
|
1565
2004
|
// 2. initialize slots
|
1566
2005
|
// 3. deallocate old slots.
|
@@ -1590,45 +2029,45 @@ class HashSetResizeHelper {
|
|
1590
2029
|
//
|
1591
2030
|
// Returns IsGrowingIntoSingleGroupApplicable result to avoid recomputation.
|
1592
2031
|
template <typename Alloc, size_t SizeOfSlot, bool TransferUsesMemcpy,
|
1593
|
-
size_t AlignOfSlot>
|
1594
|
-
ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c,
|
1595
|
-
|
2032
|
+
bool SooEnabled, size_t AlignOfSlot>
|
2033
|
+
ABSL_ATTRIBUTE_NOINLINE bool InitializeSlots(CommonFields& c, Alloc alloc,
|
2034
|
+
ctrl_t soo_slot_h2,
|
2035
|
+
size_t key_size,
|
2036
|
+
size_t value_size) {
|
1596
2037
|
assert(c.capacity());
|
1597
|
-
// Folks with custom allocators often make unwarranted assumptions about the
|
1598
|
-
// behavior of their classes vis-a-vis trivial destructability and what
|
1599
|
-
// calls they will or won't make. Avoid sampling for people with custom
|
1600
|
-
// allocators to get us out of this mess. This is not a hard guarantee but
|
1601
|
-
// a workaround while we plan the exact guarantee we want to provide.
|
1602
|
-
const size_t sample_size =
|
1603
|
-
(std::is_same<Alloc, std::allocator<char>>::value &&
|
1604
|
-
c.slot_array() == nullptr)
|
1605
|
-
? SizeOfSlot
|
1606
|
-
: 0;
|
1607
2038
|
HashtablezInfoHandle infoz =
|
1608
|
-
|
2039
|
+
ShouldSampleHashtablezInfo<Alloc>()
|
2040
|
+
? SampleHashtablezInfo<SooEnabled>(SizeOfSlot, key_size, value_size,
|
2041
|
+
old_capacity_, was_soo_,
|
2042
|
+
forced_infoz_, c)
|
2043
|
+
: HashtablezInfoHandle{};
|
1609
2044
|
|
1610
2045
|
const bool has_infoz = infoz.IsSampled();
|
1611
|
-
|
1612
|
-
|
1613
|
-
|
1614
|
-
char* mem = static_cast<char*>(
|
1615
|
-
Allocate<BackingArrayAlignment(AlignOfSlot)>(&alloc, alloc_size));
|
2046
|
+
RawHashSetLayout layout(c.capacity(), AlignOfSlot, has_infoz);
|
2047
|
+
char* mem = static_cast<char*>(Allocate<BackingArrayAlignment(AlignOfSlot)>(
|
2048
|
+
&alloc, layout.alloc_size(SizeOfSlot)));
|
1616
2049
|
const GenerationType old_generation = c.generation();
|
1617
|
-
c.set_generation_ptr(
|
1618
|
-
mem +
|
2050
|
+
c.set_generation_ptr(
|
2051
|
+
reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
|
1619
2052
|
c.set_generation(NextGeneration(old_generation));
|
1620
|
-
c.set_control(reinterpret_cast<ctrl_t*>(mem +
|
1621
|
-
c.set_slots(mem +
|
2053
|
+
c.set_control(reinterpret_cast<ctrl_t*>(mem + layout.control_offset()));
|
2054
|
+
c.set_slots(mem + layout.slot_offset());
|
1622
2055
|
ResetGrowthLeft(c);
|
1623
2056
|
|
1624
2057
|
const bool grow_single_group =
|
1625
|
-
IsGrowingIntoSingleGroupApplicable(old_capacity_,
|
1626
|
-
if (
|
2058
|
+
IsGrowingIntoSingleGroupApplicable(old_capacity_, layout.capacity());
|
2059
|
+
if (SooEnabled && was_soo_ && grow_single_group) {
|
2060
|
+
InitControlBytesAfterSoo(c.control(), soo_slot_h2, layout.capacity());
|
2061
|
+
if (TransferUsesMemcpy && had_soo_slot_) {
|
2062
|
+
TransferSlotAfterSoo(c, SizeOfSlot);
|
2063
|
+
}
|
2064
|
+
// SooEnabled implies that old_capacity_ != 0.
|
2065
|
+
} else if ((SooEnabled || old_capacity_ != 0) && grow_single_group) {
|
1627
2066
|
if (TransferUsesMemcpy) {
|
1628
|
-
GrowSizeIntoSingleGroupTransferable(c,
|
1629
|
-
DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot
|
2067
|
+
GrowSizeIntoSingleGroupTransferable(c, SizeOfSlot);
|
2068
|
+
DeallocateOld<AlignOfSlot>(alloc, SizeOfSlot);
|
1630
2069
|
} else {
|
1631
|
-
GrowIntoSingleGroupShuffleControlBytes(c.control(),
|
2070
|
+
GrowIntoSingleGroupShuffleControlBytes(c.control(), layout.capacity());
|
1632
2071
|
}
|
1633
2072
|
} else {
|
1634
2073
|
ResetCtrl(c, SizeOfSlot);
|
@@ -1636,8 +2075,8 @@ class HashSetResizeHelper {
|
|
1636
2075
|
|
1637
2076
|
c.set_has_infoz(has_infoz);
|
1638
2077
|
if (has_infoz) {
|
1639
|
-
infoz.RecordStorageChanged(c.size(),
|
1640
|
-
if (grow_single_group || old_capacity_ == 0) {
|
2078
|
+
infoz.RecordStorageChanged(c.size(), layout.capacity());
|
2079
|
+
if ((SooEnabled && was_soo_) || grow_single_group || old_capacity_ == 0) {
|
1641
2080
|
infoz.RecordRehash(0);
|
1642
2081
|
}
|
1643
2082
|
c.set_infoz(infoz);
|
@@ -1651,21 +2090,22 @@ class HashSetResizeHelper {
|
|
1651
2090
|
// PRECONDITIONS:
|
1652
2091
|
// 1. GrowIntoSingleGroupShuffleControlBytes was already called.
|
1653
2092
|
template <class PolicyTraits, class Alloc>
|
1654
|
-
void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref
|
1655
|
-
typename PolicyTraits::slot_type* old_slots) {
|
2093
|
+
void GrowSizeIntoSingleGroup(CommonFields& c, Alloc& alloc_ref) {
|
1656
2094
|
assert(old_capacity_ < Group::kWidth / 2);
|
1657
2095
|
assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
|
1658
2096
|
using slot_type = typename PolicyTraits::slot_type;
|
1659
2097
|
assert(is_single_group(c.capacity()));
|
1660
2098
|
|
1661
|
-
auto* new_slots =
|
2099
|
+
auto* new_slots = static_cast<slot_type*>(c.slot_array());
|
2100
|
+
auto* old_slots_ptr = static_cast<slot_type*>(old_slots());
|
1662
2101
|
|
1663
2102
|
size_t shuffle_bit = old_capacity_ / 2 + 1;
|
1664
2103
|
for (size_t i = 0; i < old_capacity_; ++i) {
|
1665
|
-
if (IsFull(
|
2104
|
+
if (IsFull(old_ctrl()[i])) {
|
1666
2105
|
size_t new_i = i ^ shuffle_bit;
|
1667
2106
|
SanitizerUnpoisonMemoryRegion(new_slots + new_i, sizeof(slot_type));
|
1668
|
-
PolicyTraits::transfer(&alloc_ref, new_slots + new_i,
|
2107
|
+
PolicyTraits::transfer(&alloc_ref, new_slots + new_i,
|
2108
|
+
old_slots_ptr + i);
|
1669
2109
|
}
|
1670
2110
|
}
|
1671
2111
|
PoisonSingleGroupEmptySlots(c, sizeof(slot_type));
|
@@ -1673,11 +2113,12 @@ class HashSetResizeHelper {
|
|
1673
2113
|
|
1674
2114
|
// Deallocates old backing array.
|
1675
2115
|
template <size_t AlignOfSlot, class CharAlloc>
|
1676
|
-
void DeallocateOld(CharAlloc alloc_ref, size_t slot_size
|
1677
|
-
SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_);
|
2116
|
+
void DeallocateOld(CharAlloc alloc_ref, size_t slot_size) {
|
2117
|
+
SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_);
|
2118
|
+
auto layout = RawHashSetLayout(old_capacity_, AlignOfSlot, had_infoz_);
|
1678
2119
|
Deallocate<BackingArrayAlignment(AlignOfSlot)>(
|
1679
|
-
&alloc_ref,
|
1680
|
-
|
2120
|
+
&alloc_ref, old_ctrl() - layout.control_offset(),
|
2121
|
+
layout.alloc_size(slot_size));
|
1681
2122
|
}
|
1682
2123
|
|
1683
2124
|
private:
|
@@ -1692,8 +2133,12 @@ class HashSetResizeHelper {
|
|
1692
2133
|
// Relocates control bytes and slots into new single group for
|
1693
2134
|
// transferable objects.
|
1694
2135
|
// Must be called only if IsGrowingIntoSingleGroupApplicable returned true.
|
1695
|
-
void GrowSizeIntoSingleGroupTransferable(CommonFields& c,
|
1696
|
-
|
2136
|
+
void GrowSizeIntoSingleGroupTransferable(CommonFields& c, size_t slot_size);
|
2137
|
+
|
2138
|
+
// If there was an SOO slot and slots are transferable, transfers the SOO slot
|
2139
|
+
// into the new heap allocation. Must be called only if
|
2140
|
+
// IsGrowingIntoSingleGroupApplicable returned true.
|
2141
|
+
void TransferSlotAfterSoo(CommonFields& c, size_t slot_size);
|
1697
2142
|
|
1698
2143
|
// Shuffle control bits deterministically to the next capacity.
|
1699
2144
|
// Returns offset for newly added element with given hash.
|
@@ -1726,6 +2171,13 @@ class HashSetResizeHelper {
|
|
1726
2171
|
void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* new_ctrl,
|
1727
2172
|
size_t new_capacity) const;
|
1728
2173
|
|
2174
|
+
// If the table was SOO, initializes new control bytes. `h2` is the control
|
2175
|
+
// byte corresponding to the full slot. Must be called only if
|
2176
|
+
// IsGrowingIntoSingleGroupApplicable returned true.
|
2177
|
+
// Requires: `had_soo_slot_ || h2 == ctrl_t::kEmpty`.
|
2178
|
+
void InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2,
|
2179
|
+
size_t new_capacity);
|
2180
|
+
|
1729
2181
|
// Shuffle trivially transferable slots in the way consistent with
|
1730
2182
|
// GrowIntoSingleGroupShuffleControlBytes.
|
1731
2183
|
//
|
@@ -1739,8 +2191,7 @@ class HashSetResizeHelper {
|
|
1739
2191
|
// 1. new_slots are transferred from old_slots_ consistent with
|
1740
2192
|
// GrowIntoSingleGroupShuffleControlBytes.
|
1741
2193
|
// 2. Empty new_slots are *not* poisoned.
|
1742
|
-
void GrowIntoSingleGroupShuffleTransferableSlots(void*
|
1743
|
-
void* new_slots,
|
2194
|
+
void GrowIntoSingleGroupShuffleTransferableSlots(void* new_slots,
|
1744
2195
|
size_t slot_size) const;
|
1745
2196
|
|
1746
2197
|
// Poison empty slots that were transferred using the deterministic algorithm
|
@@ -1760,11 +2211,24 @@ class HashSetResizeHelper {
|
|
1760
2211
|
}
|
1761
2212
|
}
|
1762
2213
|
|
1763
|
-
|
2214
|
+
HeapOrSoo old_heap_or_soo_;
|
1764
2215
|
size_t old_capacity_;
|
1765
2216
|
bool had_infoz_;
|
2217
|
+
bool was_soo_;
|
2218
|
+
bool had_soo_slot_;
|
2219
|
+
// Either null infoz or a pre-sampled forced infoz for SOO tables.
|
2220
|
+
HashtablezInfoHandle forced_infoz_;
|
1766
2221
|
};
|
1767
2222
|
|
2223
|
+
inline void PrepareInsertCommon(CommonFields& common) {
|
2224
|
+
common.increment_size();
|
2225
|
+
common.maybe_increment_generation_on_insert();
|
2226
|
+
}
|
2227
|
+
|
2228
|
+
// Like prepare_insert, but for the case of inserting into a full SOO table.
|
2229
|
+
size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size,
|
2230
|
+
CommonFields& common);
|
2231
|
+
|
1768
2232
|
// PolicyFunctions bundles together some information for a particular
|
1769
2233
|
// raw_hash_set<T, ...> instantiation. This information is passed to
|
1770
2234
|
// type-erased functions that want to do small amounts of type-specific
|
@@ -1772,21 +2236,29 @@ class HashSetResizeHelper {
|
|
1772
2236
|
struct PolicyFunctions {
|
1773
2237
|
size_t slot_size;
|
1774
2238
|
|
2239
|
+
// Returns the pointer to the hash function stored in the set.
|
2240
|
+
const void* (*hash_fn)(const CommonFields& common);
|
2241
|
+
|
1775
2242
|
// Returns the hash of the pointed-to slot.
|
1776
|
-
size_t (*hash_slot)(void*
|
2243
|
+
size_t (*hash_slot)(const void* hash_fn, void* slot);
|
1777
2244
|
|
1778
|
-
//
|
2245
|
+
// Transfers the contents of src_slot to dst_slot.
|
1779
2246
|
void (*transfer)(void* set, void* dst_slot, void* src_slot);
|
1780
2247
|
|
1781
|
-
//
|
2248
|
+
// Deallocates the backing store from common.
|
1782
2249
|
void (*dealloc)(CommonFields& common, const PolicyFunctions& policy);
|
2250
|
+
|
2251
|
+
// Resizes set to the new capacity.
|
2252
|
+
// Arguments are used as in raw_hash_set::resize_impl.
|
2253
|
+
void (*resize)(CommonFields& common, size_t new_capacity,
|
2254
|
+
HashtablezInfoHandle forced_infoz);
|
1783
2255
|
};
|
1784
2256
|
|
1785
2257
|
// ClearBackingArray clears the backing array, either modifying it in place,
|
1786
2258
|
// or creating a new one based on the value of "reuse".
|
1787
2259
|
// REQUIRES: c.capacity > 0
|
1788
2260
|
void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
|
1789
|
-
bool reuse);
|
2261
|
+
bool reuse, bool soo_enabled);
|
1790
2262
|
|
1791
2263
|
// Type-erased version of raw_hash_set::erase_meta_only.
|
1792
2264
|
void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size);
|
@@ -1817,9 +2289,26 @@ ABSL_ATTRIBUTE_NOINLINE void TransferRelocatable(void*, void* dst, void* src) {
|
|
1817
2289
|
memcpy(dst, src, SizeOfSlot);
|
1818
2290
|
}
|
1819
2291
|
|
1820
|
-
// Type
|
1821
|
-
void
|
1822
|
-
|
2292
|
+
// Type erased raw_hash_set::get_hash_ref_fn for the empty hash function case.
|
2293
|
+
const void* GetHashRefForEmptyHasher(const CommonFields& common);
|
2294
|
+
|
2295
|
+
// Given the hash of a value not currently in the table and the first empty
|
2296
|
+
// slot in the probe sequence, finds a viable slot index to insert it at.
|
2297
|
+
//
|
2298
|
+
// In case there's no space left, the table can be resized or rehashed
|
2299
|
+
// (for tables with deleted slots, see FindInsertPositionWithGrowthOrRehash).
|
2300
|
+
//
|
2301
|
+
// In the case of absence of deleted slots and positive growth_left, the element
|
2302
|
+
// can be inserted in the provided `target` position.
|
2303
|
+
//
|
2304
|
+
// When the table has deleted slots (according to GrowthInfo), the target
|
2305
|
+
// position will be searched one more time using `find_first_non_full`.
|
2306
|
+
//
|
2307
|
+
// REQUIRES: Table is not SOO.
|
2308
|
+
// REQUIRES: At least one non-full slot available.
|
2309
|
+
// REQUIRES: `target` is a valid empty position to insert.
|
2310
|
+
size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target,
|
2311
|
+
const PolicyFunctions& policy);
|
1823
2312
|
|
1824
2313
|
// A SwissTable.
|
1825
2314
|
//
|
@@ -1875,6 +2364,26 @@ class raw_hash_set {
|
|
1875
2364
|
using key_arg = typename KeyArgImpl::template type<K, key_type>;
|
1876
2365
|
|
1877
2366
|
private:
|
2367
|
+
// TODO(b/289225379): we could add extra SOO space inside raw_hash_set
|
2368
|
+
// after CommonFields to allow inlining larger slot_types (e.g. std::string),
|
2369
|
+
// but it's a bit complicated if we want to support incomplete mapped_type in
|
2370
|
+
// flat_hash_map. We could potentially do this for flat_hash_set and for an
|
2371
|
+
// allowlist of `mapped_type`s of flat_hash_map that includes e.g. arithmetic
|
2372
|
+
// types, strings, cords, and pairs/tuples of allowlisted types.
|
2373
|
+
constexpr static bool SooEnabled() {
|
2374
|
+
return PolicyTraits::soo_enabled() &&
|
2375
|
+
sizeof(slot_type) <= sizeof(HeapOrSoo) &&
|
2376
|
+
alignof(slot_type) <= alignof(HeapOrSoo);
|
2377
|
+
}
|
2378
|
+
|
2379
|
+
// Whether `size` fits in the SOO capacity of this table.
|
2380
|
+
bool fits_in_soo(size_t size) const {
|
2381
|
+
return SooEnabled() && size <= SooCapacity();
|
2382
|
+
}
|
2383
|
+
// Whether this table is in SOO mode or non-SOO mode.
|
2384
|
+
bool is_soo() const { return fits_in_soo(capacity()); }
|
2385
|
+
bool is_full_soo() const { return is_soo() && !empty(); }
|
2386
|
+
|
1878
2387
|
// Give an early error when key_type is not hashable/eq.
|
1879
2388
|
auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
|
1880
2389
|
auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k));
|
@@ -1928,6 +2437,7 @@ class raw_hash_set {
|
|
1928
2437
|
|
1929
2438
|
class iterator : private HashSetIteratorGenerationInfo {
|
1930
2439
|
friend class raw_hash_set;
|
2440
|
+
friend struct HashtableFreeFunctionsAccess;
|
1931
2441
|
|
1932
2442
|
public:
|
1933
2443
|
using iterator_category = std::forward_iterator_tag;
|
@@ -1958,6 +2468,7 @@ class raw_hash_set {
|
|
1958
2468
|
++ctrl_;
|
1959
2469
|
++slot_;
|
1960
2470
|
skip_empty_or_deleted();
|
2471
|
+
if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr;
|
1961
2472
|
return *this;
|
1962
2473
|
}
|
1963
2474
|
// PRECONDITION: not an end() iterator.
|
@@ -1988,22 +2499,31 @@ class raw_hash_set {
|
|
1988
2499
|
// not equal to any end iterator.
|
1989
2500
|
ABSL_ASSUME(ctrl != nullptr);
|
1990
2501
|
}
|
2502
|
+
// This constructor is used in begin() to avoid an MSan
|
2503
|
+
// use-of-uninitialized-value error. Delegating from this constructor to
|
2504
|
+
// the previous one doesn't avoid the error.
|
2505
|
+
iterator(ctrl_t* ctrl, MaybeInitializedPtr slot,
|
2506
|
+
const GenerationType* generation_ptr)
|
2507
|
+
: HashSetIteratorGenerationInfo(generation_ptr),
|
2508
|
+
ctrl_(ctrl),
|
2509
|
+
slot_(to_slot(slot.get())) {
|
2510
|
+
// This assumption helps the compiler know that any non-end iterator is
|
2511
|
+
// not equal to any end iterator.
|
2512
|
+
ABSL_ASSUME(ctrl != nullptr);
|
2513
|
+
}
|
1991
2514
|
// For end() iterators.
|
1992
2515
|
explicit iterator(const GenerationType* generation_ptr)
|
1993
2516
|
: HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {}
|
1994
2517
|
|
1995
|
-
// Fixes up `ctrl_` to point to a full by advancing
|
1996
|
-
// they reach one.
|
1997
|
-
//
|
1998
|
-
// If a sentinel is reached, we null `ctrl_` out instead.
|
2518
|
+
// Fixes up `ctrl_` to point to a full or sentinel by advancing `ctrl_` and
|
2519
|
+
// `slot_` until they reach one.
|
1999
2520
|
void skip_empty_or_deleted() {
|
2000
2521
|
while (IsEmptyOrDeleted(*ctrl_)) {
|
2001
2522
|
uint32_t shift =
|
2002
|
-
|
2523
|
+
GroupFullEmptyOrDeleted{ctrl_}.CountLeadingEmptyOrDeleted();
|
2003
2524
|
ctrl_ += shift;
|
2004
2525
|
slot_ += shift;
|
2005
2526
|
}
|
2006
|
-
if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr;
|
2007
2527
|
}
|
2008
2528
|
|
2009
2529
|
ctrl_t* control() const { return ctrl_; }
|
@@ -2091,8 +2611,9 @@ class raw_hash_set {
|
|
2091
2611
|
size_t bucket_count, const hasher& hash = hasher(),
|
2092
2612
|
const key_equal& eq = key_equal(),
|
2093
2613
|
const allocator_type& alloc = allocator_type())
|
2094
|
-
: settings_(CommonFields
|
2095
|
-
|
2614
|
+
: settings_(CommonFields::CreateDefault<SooEnabled()>(), hash, eq,
|
2615
|
+
alloc) {
|
2616
|
+
if (bucket_count > (SooEnabled() ? SooCapacity() : 0)) {
|
2096
2617
|
resize(NormalizeCapacity(bucket_count));
|
2097
2618
|
}
|
2098
2619
|
}
|
@@ -2193,22 +2714,69 @@ class raw_hash_set {
|
|
2193
2714
|
that.alloc_ref())) {}
|
2194
2715
|
|
2195
2716
|
raw_hash_set(const raw_hash_set& that, const allocator_type& a)
|
2196
|
-
: raw_hash_set(
|
2717
|
+
: raw_hash_set(GrowthToLowerboundCapacity(that.size()), that.hash_ref(),
|
2718
|
+
that.eq_ref(), a) {
|
2197
2719
|
const size_t size = that.size();
|
2198
|
-
if (size == 0)
|
2199
|
-
|
2200
|
-
|
2201
|
-
//
|
2202
|
-
|
2203
|
-
|
2204
|
-
|
2205
|
-
|
2206
|
-
emplace_at(
|
2207
|
-
|
2208
|
-
infoz()
|
2720
|
+
if (size == 0) {
|
2721
|
+
return;
|
2722
|
+
}
|
2723
|
+
// We don't use `that.is_soo()` here because `that` can have non-SOO
|
2724
|
+
// capacity but have a size that fits into SOO capacity.
|
2725
|
+
if (fits_in_soo(size)) {
|
2726
|
+
assert(size == 1);
|
2727
|
+
common().set_full_soo();
|
2728
|
+
emplace_at(soo_iterator(), *that.begin());
|
2729
|
+
const HashtablezInfoHandle infoz = try_sample_soo();
|
2730
|
+
if (infoz.IsSampled()) resize_with_soo_infoz(infoz);
|
2731
|
+
return;
|
2732
|
+
}
|
2733
|
+
assert(!that.is_soo());
|
2734
|
+
const size_t cap = capacity();
|
2735
|
+
// Note about single group tables:
|
2736
|
+
// 1. It is correct to have any order of elements.
|
2737
|
+
// 2. Order has to be non deterministic.
|
2738
|
+
// 3. We are assigning elements with arbitrary `shift` starting from
|
2739
|
+
// `capacity + shift` position.
|
2740
|
+
// 4. `shift` must be coprime with `capacity + 1` in order to be able to use
|
2741
|
+
// modular arithmetic to traverse all positions, instead if cycling
|
2742
|
+
// through a subset of positions. Odd numbers are coprime with any
|
2743
|
+
// `capacity + 1` (2^N).
|
2744
|
+
size_t offset = cap;
|
2745
|
+
const size_t shift =
|
2746
|
+
is_single_group(cap) ? (PerTableSalt(control()) | 1) : 0;
|
2747
|
+
IterateOverFullSlots(
|
2748
|
+
that.common(), that.slot_array(),
|
2749
|
+
[&](const ctrl_t* that_ctrl,
|
2750
|
+
slot_type* that_slot) ABSL_ATTRIBUTE_ALWAYS_INLINE {
|
2751
|
+
if (shift == 0) {
|
2752
|
+
// Big tables case. Position must be searched via probing.
|
2753
|
+
// The table is guaranteed to be empty, so we can do faster than
|
2754
|
+
// a full `insert`.
|
2755
|
+
const size_t hash = PolicyTraits::apply(
|
2756
|
+
HashElement{hash_ref()}, PolicyTraits::element(that_slot));
|
2757
|
+
FindInfo target = find_first_non_full_outofline(common(), hash);
|
2758
|
+
infoz().RecordInsert(hash, target.probe_length);
|
2759
|
+
offset = target.offset;
|
2760
|
+
} else {
|
2761
|
+
// Small tables case. Next position is computed via shift.
|
2762
|
+
offset = (offset + shift) & cap;
|
2763
|
+
}
|
2764
|
+
const h2_t h2 = static_cast<h2_t>(*that_ctrl);
|
2765
|
+
assert( // We rely that hash is not changed for small tables.
|
2766
|
+
H2(PolicyTraits::apply(HashElement{hash_ref()},
|
2767
|
+
PolicyTraits::element(that_slot))) == h2 &&
|
2768
|
+
"hash function value changed unexpectedly during the copy");
|
2769
|
+
SetCtrl(common(), offset, h2, sizeof(slot_type));
|
2770
|
+
emplace_at(iterator_at(offset), PolicyTraits::element(that_slot));
|
2771
|
+
common().maybe_increment_generation_on_insert();
|
2772
|
+
});
|
2773
|
+
if (shift != 0) {
|
2774
|
+
// On small table copy we do not record individual inserts.
|
2775
|
+
// RecordInsert requires hash, but it is unknown for small tables.
|
2776
|
+
infoz().RecordStorageChanged(size, cap);
|
2209
2777
|
}
|
2210
2778
|
common().set_size(size);
|
2211
|
-
|
2779
|
+
growth_info().OverwriteManyEmptyAsFull(size);
|
2212
2780
|
}
|
2213
2781
|
|
2214
2782
|
ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept(
|
@@ -2220,16 +2788,22 @@ class raw_hash_set {
|
|
2220
2788
|
// would create a nullptr functor that cannot be called.
|
2221
2789
|
// TODO(b/296061262): move instead of copying hash/eq/alloc.
|
2222
2790
|
// Note: we avoid using exchange for better generated code.
|
2223
|
-
settings_(
|
2224
|
-
|
2225
|
-
|
2791
|
+
settings_(PolicyTraits::transfer_uses_memcpy() || !that.is_full_soo()
|
2792
|
+
? std::move(that.common())
|
2793
|
+
: CommonFields{full_soo_tag_t{}},
|
2794
|
+
that.hash_ref(), that.eq_ref(), that.alloc_ref()) {
|
2795
|
+
if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) {
|
2796
|
+
transfer(soo_slot(), that.soo_slot());
|
2797
|
+
}
|
2798
|
+
that.common() = CommonFields::CreateDefault<SooEnabled()>();
|
2226
2799
|
maybe_increment_generation_or_rehash_on_move();
|
2227
2800
|
}
|
2228
2801
|
|
2229
2802
|
raw_hash_set(raw_hash_set&& that, const allocator_type& a)
|
2230
|
-
: settings_(CommonFields
|
2803
|
+
: settings_(CommonFields::CreateDefault<SooEnabled()>(), that.hash_ref(),
|
2804
|
+
that.eq_ref(), a) {
|
2231
2805
|
if (a == that.alloc_ref()) {
|
2232
|
-
|
2806
|
+
swap_common(that);
|
2233
2807
|
maybe_increment_generation_or_rehash_on_move();
|
2234
2808
|
} else {
|
2235
2809
|
move_elements_allocs_unequal(std::move(that));
|
@@ -2264,8 +2838,12 @@ class raw_hash_set {
|
|
2264
2838
|
~raw_hash_set() { destructor_impl(); }
|
2265
2839
|
|
2266
2840
|
iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
2267
|
-
|
2841
|
+
if (ABSL_PREDICT_FALSE(empty())) return end();
|
2842
|
+
if (is_soo()) return soo_iterator();
|
2843
|
+
iterator it = {control(), common().slots_union(),
|
2844
|
+
common().generation_ptr()};
|
2268
2845
|
it.skip_empty_or_deleted();
|
2846
|
+
assert(IsFull(*it.control()));
|
2269
2847
|
return it;
|
2270
2848
|
}
|
2271
2849
|
iterator end() ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
@@ -2285,7 +2863,14 @@ class raw_hash_set {
|
|
2285
2863
|
|
2286
2864
|
bool empty() const { return !size(); }
|
2287
2865
|
size_t size() const { return common().size(); }
|
2288
|
-
size_t capacity() const {
|
2866
|
+
size_t capacity() const {
|
2867
|
+
const size_t cap = common().capacity();
|
2868
|
+
// Compiler complains when using functions in assume so use local variables.
|
2869
|
+
ABSL_ATTRIBUTE_UNUSED static constexpr bool kEnabled = SooEnabled();
|
2870
|
+
ABSL_ATTRIBUTE_UNUSED static constexpr size_t kCapacity = SooCapacity();
|
2871
|
+
ABSL_ASSUME(!kEnabled || cap >= kCapacity);
|
2872
|
+
return cap;
|
2873
|
+
}
|
2289
2874
|
size_t max_size() const { return (std::numeric_limits<size_t>::max)(); }
|
2290
2875
|
|
2291
2876
|
ABSL_ATTRIBUTE_REINITIALIZES void clear() {
|
@@ -2299,9 +2884,13 @@ class raw_hash_set {
|
|
2299
2884
|
const size_t cap = capacity();
|
2300
2885
|
if (cap == 0) {
|
2301
2886
|
// Already guaranteed to be empty; so nothing to do.
|
2887
|
+
} else if (is_soo()) {
|
2888
|
+
if (!empty()) destroy(soo_slot());
|
2889
|
+
common().set_empty_soo();
|
2302
2890
|
} else {
|
2303
2891
|
destroy_slots();
|
2304
|
-
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128
|
2892
|
+
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/cap < 128,
|
2893
|
+
SooEnabled());
|
2305
2894
|
}
|
2306
2895
|
common().set_reserved_growth(0);
|
2307
2896
|
common().set_reservation_size(0);
|
@@ -2432,7 +3021,7 @@ class raw_hash_set {
|
|
2432
3021
|
std::pair<iterator, bool> emplace(Args&&... args)
|
2433
3022
|
ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
2434
3023
|
alignas(slot_type) unsigned char raw[sizeof(slot_type)];
|
2435
|
-
slot_type* slot =
|
3024
|
+
slot_type* slot = to_slot(&raw);
|
2436
3025
|
|
2437
3026
|
construct(slot, std::forward<Args>(args)...);
|
2438
3027
|
const auto& elem = PolicyTraits::element(slot);
|
@@ -2496,11 +3085,11 @@ class raw_hash_set {
|
|
2496
3085
|
F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
2497
3086
|
auto res = find_or_prepare_insert(key);
|
2498
3087
|
if (res.second) {
|
2499
|
-
slot_type* slot =
|
3088
|
+
slot_type* slot = res.first.slot();
|
2500
3089
|
std::forward<F>(f)(constructor(&alloc_ref(), &slot));
|
2501
3090
|
assert(!slot);
|
2502
3091
|
}
|
2503
|
-
return
|
3092
|
+
return res.first;
|
2504
3093
|
}
|
2505
3094
|
|
2506
3095
|
// Extension API: support for heterogeneous keys.
|
@@ -2524,7 +3113,7 @@ class raw_hash_set {
|
|
2524
3113
|
// this method returns void to reduce algorithmic complexity to O(1). The
|
2525
3114
|
// iterator is invalidated, so any increment should be done before calling
|
2526
3115
|
// erase. In order to erase while iterating across a map, use the following
|
2527
|
-
// idiom (which also works for standard containers):
|
3116
|
+
// idiom (which also works for some standard containers):
|
2528
3117
|
//
|
2529
3118
|
// for (auto it = m.begin(), end = m.end(); it != end;) {
|
2530
3119
|
// // `erase()` will invalidate `it`, so advance `it` first.
|
@@ -2540,7 +3129,11 @@ class raw_hash_set {
|
|
2540
3129
|
void erase(iterator it) {
|
2541
3130
|
AssertIsFull(it.control(), it.generation(), it.generation_ptr(), "erase()");
|
2542
3131
|
destroy(it.slot());
|
2543
|
-
|
3132
|
+
if (is_soo()) {
|
3133
|
+
common().set_empty_soo();
|
3134
|
+
} else {
|
3135
|
+
erase_meta_only(it);
|
3136
|
+
}
|
2544
3137
|
}
|
2545
3138
|
|
2546
3139
|
iterator erase(const_iterator first,
|
@@ -2548,12 +3141,19 @@ class raw_hash_set {
|
|
2548
3141
|
// We check for empty first because ClearBackingArray requires that
|
2549
3142
|
// capacity() > 0 as a precondition.
|
2550
3143
|
if (empty()) return end();
|
3144
|
+
if (first == last) return last.inner_;
|
3145
|
+
if (is_soo()) {
|
3146
|
+
destroy(soo_slot());
|
3147
|
+
common().set_empty_soo();
|
3148
|
+
return end();
|
3149
|
+
}
|
2551
3150
|
if (first == begin() && last == end()) {
|
2552
3151
|
// TODO(ezb): we access control bytes in destroy_slots so it could make
|
2553
3152
|
// sense to combine destroy_slots and ClearBackingArray to avoid cache
|
2554
3153
|
// misses when the table is large. Note that we also do this in clear().
|
2555
3154
|
destroy_slots();
|
2556
|
-
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true
|
3155
|
+
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/true,
|
3156
|
+
SooEnabled());
|
2557
3157
|
common().set_reserved_growth(common().reservation_size());
|
2558
3158
|
return end();
|
2559
3159
|
}
|
@@ -2568,13 +3168,21 @@ class raw_hash_set {
|
|
2568
3168
|
template <typename H, typename E>
|
2569
3169
|
void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT
|
2570
3170
|
assert(this != &src);
|
3171
|
+
// Returns whether insertion took place.
|
3172
|
+
const auto insert_slot = [this](slot_type* src_slot) {
|
3173
|
+
return PolicyTraits::apply(InsertSlot<false>{*this, std::move(*src_slot)},
|
3174
|
+
PolicyTraits::element(src_slot))
|
3175
|
+
.second;
|
3176
|
+
};
|
3177
|
+
|
3178
|
+
if (src.is_soo()) {
|
3179
|
+
if (src.empty()) return;
|
3180
|
+
if (insert_slot(src.soo_slot())) src.common().set_empty_soo();
|
3181
|
+
return;
|
3182
|
+
}
|
2571
3183
|
for (auto it = src.begin(), e = src.end(); it != e;) {
|
2572
3184
|
auto next = std::next(it);
|
2573
|
-
if (
|
2574
|
-
PolicyTraits::element(it.slot()))
|
2575
|
-
.second) {
|
2576
|
-
src.erase_meta_only(it);
|
2577
|
-
}
|
3185
|
+
if (insert_slot(it.slot())) src.erase_meta_only(it);
|
2578
3186
|
it = next;
|
2579
3187
|
}
|
2580
3188
|
}
|
@@ -2588,7 +3196,11 @@ class raw_hash_set {
|
|
2588
3196
|
AssertIsFull(position.control(), position.inner_.generation(),
|
2589
3197
|
position.inner_.generation_ptr(), "extract()");
|
2590
3198
|
auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.slot());
|
2591
|
-
|
3199
|
+
if (is_soo()) {
|
3200
|
+
common().set_empty_soo();
|
3201
|
+
} else {
|
3202
|
+
erase_meta_only(position);
|
3203
|
+
}
|
2592
3204
|
return node;
|
2593
3205
|
}
|
2594
3206
|
|
@@ -2605,7 +3217,7 @@ class raw_hash_set {
|
|
2605
3217
|
IsNoThrowSwappable<allocator_type>(
|
2606
3218
|
typename AllocTraits::propagate_on_container_swap{})) {
|
2607
3219
|
using std::swap;
|
2608
|
-
|
3220
|
+
swap_common(that);
|
2609
3221
|
swap(hash_ref(), that.hash_ref());
|
2610
3222
|
swap(eq_ref(), that.eq_ref());
|
2611
3223
|
SwapAlloc(alloc_ref(), that.alloc_ref(),
|
@@ -2613,17 +3225,41 @@ class raw_hash_set {
|
|
2613
3225
|
}
|
2614
3226
|
|
2615
3227
|
void rehash(size_t n) {
|
2616
|
-
|
2617
|
-
if (n == 0
|
2618
|
-
|
2619
|
-
|
3228
|
+
const size_t cap = capacity();
|
3229
|
+
if (n == 0) {
|
3230
|
+
if (cap == 0 || is_soo()) return;
|
3231
|
+
if (empty()) {
|
3232
|
+
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false,
|
3233
|
+
SooEnabled());
|
3234
|
+
return;
|
3235
|
+
}
|
3236
|
+
if (fits_in_soo(size())) {
|
3237
|
+
// When the table is already sampled, we keep it sampled.
|
3238
|
+
if (infoz().IsSampled()) {
|
3239
|
+
const size_t kInitialSampledCapacity = NextCapacity(SooCapacity());
|
3240
|
+
if (capacity() > kInitialSampledCapacity) {
|
3241
|
+
resize(kInitialSampledCapacity);
|
3242
|
+
}
|
3243
|
+
// This asserts that we didn't lose sampling coverage in `resize`.
|
3244
|
+
assert(infoz().IsSampled());
|
3245
|
+
return;
|
3246
|
+
}
|
3247
|
+
alignas(slot_type) unsigned char slot_space[sizeof(slot_type)];
|
3248
|
+
slot_type* tmp_slot = to_slot(slot_space);
|
3249
|
+
transfer(tmp_slot, begin().slot());
|
3250
|
+
ClearBackingArray(common(), GetPolicyFunctions(), /*reuse=*/false,
|
3251
|
+
SooEnabled());
|
3252
|
+
transfer(soo_slot(), tmp_slot);
|
3253
|
+
common().set_full_soo();
|
3254
|
+
return;
|
3255
|
+
}
|
2620
3256
|
}
|
2621
3257
|
|
2622
3258
|
// bitor is a faster way of doing `max` here. We will round up to the next
|
2623
3259
|
// power-of-2-minus-1, so bitor is good enough.
|
2624
3260
|
auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size()));
|
2625
3261
|
// n == 0 unconditionally rehashes as per the standard.
|
2626
|
-
if (n == 0 || m >
|
3262
|
+
if (n == 0 || m > cap) {
|
2627
3263
|
resize(m);
|
2628
3264
|
|
2629
3265
|
// This is after resize, to ensure that we have completed the allocation
|
@@ -2633,7 +3269,9 @@ class raw_hash_set {
|
|
2633
3269
|
}
|
2634
3270
|
|
2635
3271
|
void reserve(size_t n) {
|
2636
|
-
|
3272
|
+
const size_t max_size_before_growth =
|
3273
|
+
is_soo() ? SooCapacity() : size() + growth_left();
|
3274
|
+
if (n > max_size_before_growth) {
|
2637
3275
|
size_t m = GrowthToLowerboundCapacity(n);
|
2638
3276
|
resize(NormalizeCapacity(m));
|
2639
3277
|
|
@@ -2666,6 +3304,7 @@ class raw_hash_set {
|
|
2666
3304
|
// specific benchmarks indicating its importance.
|
2667
3305
|
template <class K = key_type>
|
2668
3306
|
void prefetch(const key_arg<K>& key) const {
|
3307
|
+
if (SooEnabled() ? is_soo() : capacity() == 0) return;
|
2669
3308
|
(void)key;
|
2670
3309
|
// Avoid probing if we won't be able to prefetch the addresses received.
|
2671
3310
|
#ifdef ABSL_HAVE_PREFETCH
|
@@ -2686,26 +3325,16 @@ class raw_hash_set {
|
|
2686
3325
|
template <class K = key_type>
|
2687
3326
|
iterator find(const key_arg<K>& key,
|
2688
3327
|
size_t hash) ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
2689
|
-
|
2690
|
-
|
2691
|
-
|
2692
|
-
while (true) {
|
2693
|
-
Group g{ctrl + seq.offset()};
|
2694
|
-
for (uint32_t i : g.Match(H2(hash))) {
|
2695
|
-
if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
|
2696
|
-
EqualElement<K>{key, eq_ref()},
|
2697
|
-
PolicyTraits::element(slot_ptr + seq.offset(i)))))
|
2698
|
-
return iterator_at(seq.offset(i));
|
2699
|
-
}
|
2700
|
-
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
|
2701
|
-
seq.next();
|
2702
|
-
assert(seq.index() <= capacity() && "full table!");
|
2703
|
-
}
|
3328
|
+
AssertHashEqConsistent(key);
|
3329
|
+
if (is_soo()) return find_soo(key);
|
3330
|
+
return find_non_soo(key, hash);
|
2704
3331
|
}
|
2705
3332
|
template <class K = key_type>
|
2706
3333
|
iterator find(const key_arg<K>& key) ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
3334
|
+
AssertHashEqConsistent(key);
|
3335
|
+
if (is_soo()) return find_soo(key);
|
2707
3336
|
prefetch_heap_block();
|
2708
|
-
return
|
3337
|
+
return find_non_soo(key, hash_ref()(key));
|
2709
3338
|
}
|
2710
3339
|
|
2711
3340
|
template <class K = key_type>
|
@@ -2716,8 +3345,7 @@ class raw_hash_set {
|
|
2716
3345
|
template <class K = key_type>
|
2717
3346
|
const_iterator find(const key_arg<K>& key) const
|
2718
3347
|
ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
2719
|
-
|
2720
|
-
return find(key, hash_ref()(key));
|
3348
|
+
return const_cast<raw_hash_set*>(this)->find(key);
|
2721
3349
|
}
|
2722
3350
|
|
2723
3351
|
template <class K = key_type>
|
@@ -2791,6 +3419,8 @@ class raw_hash_set {
|
|
2791
3419
|
friend struct absl::container_internal::hashtable_debug_internal::
|
2792
3420
|
HashtableDebugAccess;
|
2793
3421
|
|
3422
|
+
friend struct absl::container_internal::HashtableFreeFunctionsAccess;
|
3423
|
+
|
2794
3424
|
struct FindElement {
|
2795
3425
|
template <class K, class... Args>
|
2796
3426
|
const_iterator operator()(const K& key, Args&&...) const {
|
@@ -2824,7 +3454,7 @@ class raw_hash_set {
|
|
2824
3454
|
if (res.second) {
|
2825
3455
|
s.emplace_at(res.first, std::forward<Args>(args)...);
|
2826
3456
|
}
|
2827
|
-
return
|
3457
|
+
return res;
|
2828
3458
|
}
|
2829
3459
|
raw_hash_set& s;
|
2830
3460
|
};
|
@@ -2835,11 +3465,11 @@ class raw_hash_set {
|
|
2835
3465
|
std::pair<iterator, bool> operator()(const K& key, Args&&...) && {
|
2836
3466
|
auto res = s.find_or_prepare_insert(key);
|
2837
3467
|
if (res.second) {
|
2838
|
-
s.transfer(
|
3468
|
+
s.transfer(res.first.slot(), &slot);
|
2839
3469
|
} else if (do_destroy) {
|
2840
3470
|
s.destroy(&slot);
|
2841
3471
|
}
|
2842
|
-
return
|
3472
|
+
return res;
|
2843
3473
|
}
|
2844
3474
|
raw_hash_set& s;
|
2845
3475
|
// Constructed slot. Either moved into place or destroyed.
|
@@ -2858,17 +3488,55 @@ class raw_hash_set {
|
|
2858
3488
|
PolicyTraits::transfer(&alloc_ref(), to, from);
|
2859
3489
|
}
|
2860
3490
|
|
2861
|
-
|
2862
|
-
|
3491
|
+
// TODO(b/289225379): consider having a helper class that has the impls for
|
3492
|
+
// SOO functionality.
|
3493
|
+
template <class K = key_type>
|
3494
|
+
iterator find_soo(const key_arg<K>& key) {
|
3495
|
+
assert(is_soo());
|
3496
|
+
return empty() || !PolicyTraits::apply(EqualElement<K>{key, eq_ref()},
|
3497
|
+
PolicyTraits::element(soo_slot()))
|
3498
|
+
? end()
|
3499
|
+
: soo_iterator();
|
3500
|
+
}
|
3501
|
+
|
3502
|
+
template <class K = key_type>
|
3503
|
+
iterator find_non_soo(const key_arg<K>& key, size_t hash) {
|
3504
|
+
assert(!is_soo());
|
3505
|
+
auto seq = probe(common(), hash);
|
2863
3506
|
const ctrl_t* ctrl = control();
|
2864
|
-
|
2865
|
-
|
2866
|
-
|
2867
|
-
|
3507
|
+
while (true) {
|
3508
|
+
Group g{ctrl + seq.offset()};
|
3509
|
+
for (uint32_t i : g.Match(H2(hash))) {
|
3510
|
+
if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
|
3511
|
+
EqualElement<K>{key, eq_ref()},
|
3512
|
+
PolicyTraits::element(slot_array() + seq.offset(i)))))
|
3513
|
+
return iterator_at(seq.offset(i));
|
2868
3514
|
}
|
3515
|
+
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end();
|
3516
|
+
seq.next();
|
3517
|
+
assert(seq.index() <= capacity() && "full table!");
|
2869
3518
|
}
|
2870
3519
|
}
|
2871
3520
|
|
3521
|
+
// Conditionally samples hashtablez for SOO tables. This should be called on
|
3522
|
+
// insertion into an empty SOO table and in copy construction when the size
|
3523
|
+
// can fit in SOO capacity.
|
3524
|
+
inline HashtablezInfoHandle try_sample_soo() {
|
3525
|
+
assert(is_soo());
|
3526
|
+
if (!ShouldSampleHashtablezInfo<CharAlloc>()) return HashtablezInfoHandle{};
|
3527
|
+
return Sample(sizeof(slot_type), sizeof(key_type), sizeof(value_type),
|
3528
|
+
SooCapacity());
|
3529
|
+
}
|
3530
|
+
|
3531
|
+
inline void destroy_slots() {
|
3532
|
+
assert(!is_soo());
|
3533
|
+
if (PolicyTraits::template destroy_is_trivial<Alloc>()) return;
|
3534
|
+
IterateOverFullSlots(
|
3535
|
+
common(), slot_array(),
|
3536
|
+
[&](const ctrl_t*, slot_type* slot)
|
3537
|
+
ABSL_ATTRIBUTE_ALWAYS_INLINE { this->destroy(slot); });
|
3538
|
+
}
|
3539
|
+
|
2872
3540
|
inline void dealloc() {
|
2873
3541
|
assert(capacity() != 0);
|
2874
3542
|
// Unpoison before returning the memory to the allocator.
|
@@ -2881,6 +3549,12 @@ class raw_hash_set {
|
|
2881
3549
|
|
2882
3550
|
inline void destructor_impl() {
|
2883
3551
|
if (capacity() == 0) return;
|
3552
|
+
if (is_soo()) {
|
3553
|
+
if (!empty()) {
|
3554
|
+
ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(destroy(soo_slot()));
|
3555
|
+
}
|
3556
|
+
return;
|
3557
|
+
}
|
2884
3558
|
destroy_slots();
|
2885
3559
|
dealloc();
|
2886
3560
|
}
|
@@ -2890,10 +3564,16 @@ class raw_hash_set {
|
|
2890
3564
|
// This merely updates the pertinent control byte. This can be used in
|
2891
3565
|
// conjunction with Policy::transfer to move the object to another place.
|
2892
3566
|
void erase_meta_only(const_iterator it) {
|
3567
|
+
assert(!is_soo());
|
2893
3568
|
EraseMetaOnly(common(), static_cast<size_t>(it.control() - control()),
|
2894
3569
|
sizeof(slot_type));
|
2895
3570
|
}
|
2896
3571
|
|
3572
|
+
size_t hash_of(slot_type* slot) const {
|
3573
|
+
return PolicyTraits::apply(HashElement{hash_ref()},
|
3574
|
+
PolicyTraits::element(slot));
|
3575
|
+
}
|
3576
|
+
|
2897
3577
|
// Resizes table to the new capacity and move all elements to the new
|
2898
3578
|
// positions accordingly.
|
2899
3579
|
//
|
@@ -2902,143 +3582,165 @@ class raw_hash_set {
|
|
2902
3582
|
// HashSetResizeHelper::FindFirstNonFullAfterResize(
|
2903
3583
|
// common(), old_capacity, hash)
|
2904
3584
|
// can be called right after `resize`.
|
2905
|
-
|
3585
|
+
void resize(size_t new_capacity) {
|
3586
|
+
raw_hash_set::resize_impl(common(), new_capacity, HashtablezInfoHandle{});
|
3587
|
+
}
|
3588
|
+
|
3589
|
+
// As above, except that we also accept a pre-sampled, forced infoz for
|
3590
|
+
// SOO tables, since they need to switch from SOO to heap in order to
|
3591
|
+
// store the infoz.
|
3592
|
+
void resize_with_soo_infoz(HashtablezInfoHandle forced_infoz) {
|
3593
|
+
assert(forced_infoz.IsSampled());
|
3594
|
+
raw_hash_set::resize_impl(common(), NextCapacity(SooCapacity()),
|
3595
|
+
forced_infoz);
|
3596
|
+
}
|
3597
|
+
|
3598
|
+
// Resizes set to the new capacity.
|
3599
|
+
// It is a static function in order to use its pointer in GetPolicyFunctions.
|
3600
|
+
ABSL_ATTRIBUTE_NOINLINE static void resize_impl(
|
3601
|
+
CommonFields& common, size_t new_capacity,
|
3602
|
+
HashtablezInfoHandle forced_infoz) {
|
3603
|
+
raw_hash_set* set = reinterpret_cast<raw_hash_set*>(&common);
|
2906
3604
|
assert(IsValidCapacity(new_capacity));
|
2907
|
-
|
2908
|
-
|
2909
|
-
|
3605
|
+
assert(!set->fits_in_soo(new_capacity));
|
3606
|
+
const bool was_soo = set->is_soo();
|
3607
|
+
const bool had_soo_slot = was_soo && !set->empty();
|
3608
|
+
const ctrl_t soo_slot_h2 =
|
3609
|
+
had_soo_slot ? static_cast<ctrl_t>(H2(set->hash_of(set->soo_slot())))
|
3610
|
+
: ctrl_t::kEmpty;
|
3611
|
+
HashSetResizeHelper resize_helper(common, was_soo, had_soo_slot,
|
3612
|
+
forced_infoz);
|
3613
|
+
// Initialize HashSetResizeHelper::old_heap_or_soo_. We can't do this in
|
3614
|
+
// HashSetResizeHelper constructor because it can't transfer slots when
|
3615
|
+
// transfer_uses_memcpy is false.
|
3616
|
+
// TODO(b/289225379): try to handle more of the SOO cases inside
|
3617
|
+
// InitializeSlots. See comment on cl/555990034 snapshot #63.
|
3618
|
+
if (PolicyTraits::transfer_uses_memcpy() || !had_soo_slot) {
|
3619
|
+
resize_helper.old_heap_or_soo() = common.heap_or_soo();
|
3620
|
+
} else {
|
3621
|
+
set->transfer(set->to_slot(resize_helper.old_soo_data()),
|
3622
|
+
set->soo_slot());
|
3623
|
+
}
|
3624
|
+
common.set_capacity(new_capacity);
|
2910
3625
|
// Note that `InitializeSlots` does different number initialization steps
|
2911
3626
|
// depending on the values of `transfer_uses_memcpy` and capacities.
|
2912
3627
|
// Refer to the comment in `InitializeSlots` for more details.
|
2913
3628
|
const bool grow_single_group =
|
2914
3629
|
resize_helper.InitializeSlots<CharAlloc, sizeof(slot_type),
|
2915
3630
|
PolicyTraits::transfer_uses_memcpy(),
|
2916
|
-
alignof(slot_type)>(
|
2917
|
-
common(),
|
2918
|
-
|
3631
|
+
SooEnabled(), alignof(slot_type)>(
|
3632
|
+
common, CharAlloc(set->alloc_ref()), soo_slot_h2, sizeof(key_type),
|
3633
|
+
sizeof(value_type));
|
2919
3634
|
|
2920
|
-
|
3635
|
+
// In the SooEnabled() case, capacity is never 0 so we don't check.
|
3636
|
+
if (!SooEnabled() && resize_helper.old_capacity() == 0) {
|
2921
3637
|
// InitializeSlots did all the work including infoz().RecordRehash().
|
2922
3638
|
return;
|
2923
3639
|
}
|
3640
|
+
assert(resize_helper.old_capacity() > 0);
|
3641
|
+
// Nothing more to do in this case.
|
3642
|
+
if (was_soo && !had_soo_slot) return;
|
2924
3643
|
|
3644
|
+
slot_type* new_slots = set->slot_array();
|
2925
3645
|
if (grow_single_group) {
|
2926
3646
|
if (PolicyTraits::transfer_uses_memcpy()) {
|
2927
3647
|
// InitializeSlots did all the work.
|
2928
3648
|
return;
|
2929
3649
|
}
|
2930
|
-
|
2931
|
-
|
2932
|
-
|
2933
|
-
|
3650
|
+
if (was_soo) {
|
3651
|
+
set->transfer(new_slots + resize_helper.SooSlotIndex(),
|
3652
|
+
to_slot(resize_helper.old_soo_data()));
|
3653
|
+
return;
|
3654
|
+
} else {
|
3655
|
+
// We want GrowSizeIntoSingleGroup to be called here in order to make
|
3656
|
+
// InitializeSlots not depend on PolicyTraits.
|
3657
|
+
resize_helper.GrowSizeIntoSingleGroup<PolicyTraits>(common,
|
3658
|
+
set->alloc_ref());
|
3659
|
+
}
|
2934
3660
|
} else {
|
2935
3661
|
// InitializeSlots prepares control bytes to correspond to empty table.
|
2936
|
-
auto
|
2937
|
-
|
2938
|
-
|
2939
|
-
|
2940
|
-
|
2941
|
-
|
2942
|
-
|
2943
|
-
|
2944
|
-
|
2945
|
-
|
2946
|
-
|
3662
|
+
const auto insert_slot = [&](slot_type* slot) {
|
3663
|
+
size_t hash = PolicyTraits::apply(HashElement{set->hash_ref()},
|
3664
|
+
PolicyTraits::element(slot));
|
3665
|
+
auto target = find_first_non_full(common, hash);
|
3666
|
+
SetCtrl(common, target.offset, H2(hash), sizeof(slot_type));
|
3667
|
+
set->transfer(new_slots + target.offset, slot);
|
3668
|
+
return target.probe_length;
|
3669
|
+
};
|
3670
|
+
if (was_soo) {
|
3671
|
+
insert_slot(to_slot(resize_helper.old_soo_data()));
|
3672
|
+
return;
|
3673
|
+
} else {
|
3674
|
+
auto* old_slots = static_cast<slot_type*>(resize_helper.old_slots());
|
3675
|
+
size_t total_probe_length = 0;
|
3676
|
+
for (size_t i = 0; i != resize_helper.old_capacity(); ++i) {
|
3677
|
+
if (IsFull(resize_helper.old_ctrl()[i])) {
|
3678
|
+
total_probe_length += insert_slot(old_slots + i);
|
3679
|
+
}
|
2947
3680
|
}
|
3681
|
+
common.infoz().RecordRehash(total_probe_length);
|
2948
3682
|
}
|
2949
|
-
infoz().RecordRehash(total_probe_length);
|
2950
3683
|
}
|
2951
|
-
resize_helper.DeallocateOld<alignof(slot_type)>(
|
2952
|
-
|
2953
|
-
const_cast<std::remove_const_t<slot_type>*>(old_slots));
|
3684
|
+
resize_helper.DeallocateOld<alignof(slot_type)>(CharAlloc(set->alloc_ref()),
|
3685
|
+
sizeof(slot_type));
|
2954
3686
|
}
|
2955
3687
|
|
2956
|
-
//
|
2957
|
-
//
|
2958
|
-
|
2959
|
-
inline void drop_deletes_without_resize() {
|
2960
|
-
// Stack-allocate space for swapping elements.
|
2961
|
-
alignas(slot_type) unsigned char tmp[sizeof(slot_type)];
|
2962
|
-
DropDeletesWithoutResize(common(), GetPolicyFunctions(), tmp);
|
2963
|
-
}
|
3688
|
+
// Casting directly from e.g. char* to slot_type* can cause compilation errors
|
3689
|
+
// on objective-C. This function converts to void* first, avoiding the issue.
|
3690
|
+
static slot_type* to_slot(void* buf) { return static_cast<slot_type*>(buf); }
|
2964
3691
|
|
2965
|
-
//
|
2966
|
-
|
2967
|
-
|
2968
|
-
|
2969
|
-
|
2970
|
-
void rehash_and_grow_if_necessary() {
|
2971
|
-
const size_t cap = capacity();
|
2972
|
-
if (cap > Group::kWidth &&
|
2973
|
-
// Do these calculations in 64-bit to avoid overflow.
|
2974
|
-
size() * uint64_t{32} <= cap * uint64_t{25}) {
|
2975
|
-
// Squash DELETED without growing if there is enough capacity.
|
2976
|
-
//
|
2977
|
-
// Rehash in place if the current size is <= 25/32 of capacity.
|
2978
|
-
// Rationale for such a high factor: 1) drop_deletes_without_resize() is
|
2979
|
-
// faster than resize, and 2) it takes quite a bit of work to add
|
2980
|
-
// tombstones. In the worst case, seems to take approximately 4
|
2981
|
-
// insert/erase pairs to create a single tombstone and so if we are
|
2982
|
-
// rehashing because of tombstones, we can afford to rehash-in-place as
|
2983
|
-
// long as we are reclaiming at least 1/8 the capacity without doing more
|
2984
|
-
// than 2X the work. (Where "work" is defined to be size() for rehashing
|
2985
|
-
// or rehashing in place, and 1 for an insert or erase.) But rehashing in
|
2986
|
-
// place is faster per operation than inserting or even doubling the size
|
2987
|
-
// of the table, so we actually afford to reclaim even less space from a
|
2988
|
-
// resize-in-place. The decision is to rehash in place if we can reclaim
|
2989
|
-
// at about 1/8th of the usable capacity (specifically 3/28 of the
|
2990
|
-
// capacity) which means that the total cost of rehashing will be a small
|
2991
|
-
// fraction of the total work.
|
2992
|
-
//
|
2993
|
-
// Here is output of an experiment using the BM_CacheInSteadyState
|
2994
|
-
// benchmark running the old case (where we rehash-in-place only if we can
|
2995
|
-
// reclaim at least 7/16*capacity) vs. this code (which rehashes in place
|
2996
|
-
// if we can recover 3/32*capacity).
|
2997
|
-
//
|
2998
|
-
// Note that although in the worst-case number of rehashes jumped up from
|
2999
|
-
// 15 to 190, but the number of operations per second is almost the same.
|
3000
|
-
//
|
3001
|
-
// Abridged output of running BM_CacheInSteadyState benchmark from
|
3002
|
-
// raw_hash_set_benchmark. N is the number of insert/erase operations.
|
3003
|
-
//
|
3004
|
-
// | OLD (recover >= 7/16 | NEW (recover >= 3/32)
|
3005
|
-
// size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes
|
3006
|
-
// 448 | 145284 0.44 18 | 140118 0.44 19
|
3007
|
-
// 493 | 152546 0.24 11 | 151417 0.48 28
|
3008
|
-
// 538 | 151439 0.26 11 | 151152 0.53 38
|
3009
|
-
// 583 | 151765 0.28 11 | 150572 0.57 50
|
3010
|
-
// 628 | 150241 0.31 11 | 150853 0.61 66
|
3011
|
-
// 672 | 149602 0.33 12 | 150110 0.66 90
|
3012
|
-
// 717 | 149998 0.35 12 | 149531 0.70 129
|
3013
|
-
// 762 | 149836 0.37 13 | 148559 0.74 190
|
3014
|
-
// 807 | 149736 0.39 14 | 151107 0.39 14
|
3015
|
-
// 852 | 150204 0.42 15 | 151019 0.42 15
|
3016
|
-
drop_deletes_without_resize();
|
3692
|
+
// Requires that lhs does not have a full SOO slot.
|
3693
|
+
static void move_common(bool that_is_full_soo, allocator_type& rhs_alloc,
|
3694
|
+
CommonFields& lhs, CommonFields&& rhs) {
|
3695
|
+
if (PolicyTraits::transfer_uses_memcpy() || !that_is_full_soo) {
|
3696
|
+
lhs = std::move(rhs);
|
3017
3697
|
} else {
|
3018
|
-
|
3019
|
-
|
3698
|
+
lhs.move_non_heap_or_soo_fields(rhs);
|
3699
|
+
// TODO(b/303305702): add reentrancy guard.
|
3700
|
+
PolicyTraits::transfer(&rhs_alloc, to_slot(lhs.soo_data()),
|
3701
|
+
to_slot(rhs.soo_data()));
|
3020
3702
|
}
|
3021
3703
|
}
|
3022
3704
|
|
3705
|
+
// Swaps common fields making sure to avoid memcpy'ing a full SOO slot if we
|
3706
|
+
// aren't allowed to do so.
|
3707
|
+
void swap_common(raw_hash_set& that) {
|
3708
|
+
using std::swap;
|
3709
|
+
if (PolicyTraits::transfer_uses_memcpy()) {
|
3710
|
+
swap(common(), that.common());
|
3711
|
+
return;
|
3712
|
+
}
|
3713
|
+
CommonFields tmp = CommonFields::CreateDefault<SooEnabled()>();
|
3714
|
+
const bool that_is_full_soo = that.is_full_soo();
|
3715
|
+
move_common(that_is_full_soo, that.alloc_ref(), tmp,
|
3716
|
+
std::move(that.common()));
|
3717
|
+
move_common(is_full_soo(), alloc_ref(), that.common(), std::move(common()));
|
3718
|
+
move_common(that_is_full_soo, that.alloc_ref(), common(), std::move(tmp));
|
3719
|
+
}
|
3720
|
+
|
3023
3721
|
void maybe_increment_generation_or_rehash_on_move() {
|
3024
|
-
|
3722
|
+
if (!SwisstableGenerationsEnabled() || capacity() == 0 || is_soo()) {
|
3723
|
+
return;
|
3724
|
+
}
|
3725
|
+
common().increment_generation();
|
3025
3726
|
if (!empty() && common().should_rehash_for_bug_detection_on_move()) {
|
3026
3727
|
resize(capacity());
|
3027
3728
|
}
|
3028
3729
|
}
|
3029
3730
|
|
3030
|
-
template<bool propagate_alloc>
|
3731
|
+
template <bool propagate_alloc>
|
3031
3732
|
raw_hash_set& assign_impl(raw_hash_set&& that) {
|
3032
3733
|
// We don't bother checking for this/that aliasing. We just need to avoid
|
3033
3734
|
// breaking the invariants in that case.
|
3034
3735
|
destructor_impl();
|
3035
|
-
|
3736
|
+
move_common(that.is_full_soo(), that.alloc_ref(), common(),
|
3737
|
+
std::move(that.common()));
|
3036
3738
|
// TODO(b/296061262): move instead of copying hash/eq/alloc.
|
3037
3739
|
hash_ref() = that.hash_ref();
|
3038
3740
|
eq_ref() = that.eq_ref();
|
3039
3741
|
CopyAlloc(alloc_ref(), that.alloc_ref(),
|
3040
3742
|
std::integral_constant<bool, propagate_alloc>());
|
3041
|
-
that.common() = CommonFields
|
3743
|
+
that.common() = CommonFields::CreateDefault<SooEnabled()>();
|
3042
3744
|
maybe_increment_generation_or_rehash_on_move();
|
3043
3745
|
return *this;
|
3044
3746
|
}
|
@@ -3051,8 +3753,8 @@ class raw_hash_set {
|
|
3051
3753
|
insert(std::move(PolicyTraits::element(it.slot())));
|
3052
3754
|
that.destroy(it.slot());
|
3053
3755
|
}
|
3054
|
-
that.dealloc();
|
3055
|
-
that.common() = CommonFields
|
3756
|
+
if (!that.is_soo()) that.dealloc();
|
3757
|
+
that.common() = CommonFields::CreateDefault<SooEnabled()>();
|
3056
3758
|
maybe_increment_generation_or_rehash_on_move();
|
3057
3759
|
return *this;
|
3058
3760
|
}
|
@@ -3078,12 +3780,30 @@ class raw_hash_set {
|
|
3078
3780
|
return move_elements_allocs_unequal(std::move(that));
|
3079
3781
|
}
|
3080
3782
|
|
3081
|
-
protected:
|
3082
|
-
// Attempts to find `key` in the table; if it isn't found, returns a slot that
|
3083
|
-
// the value can be inserted into, with the control byte already set to
|
3084
|
-
// `key`'s H2.
|
3085
3783
|
template <class K>
|
3086
|
-
std::pair<
|
3784
|
+
std::pair<iterator, bool> find_or_prepare_insert_soo(const K& key) {
|
3785
|
+
if (empty()) {
|
3786
|
+
const HashtablezInfoHandle infoz = try_sample_soo();
|
3787
|
+
if (infoz.IsSampled()) {
|
3788
|
+
resize_with_soo_infoz(infoz);
|
3789
|
+
} else {
|
3790
|
+
common().set_full_soo();
|
3791
|
+
return {soo_iterator(), true};
|
3792
|
+
}
|
3793
|
+
} else if (PolicyTraits::apply(EqualElement<K>{key, eq_ref()},
|
3794
|
+
PolicyTraits::element(soo_slot()))) {
|
3795
|
+
return {soo_iterator(), false};
|
3796
|
+
} else {
|
3797
|
+
resize(NextCapacity(SooCapacity()));
|
3798
|
+
}
|
3799
|
+
const size_t index =
|
3800
|
+
PrepareInsertAfterSoo(hash_ref()(key), sizeof(slot_type), common());
|
3801
|
+
return {iterator_at(index), true};
|
3802
|
+
}
|
3803
|
+
|
3804
|
+
template <class K>
|
3805
|
+
std::pair<iterator, bool> find_or_prepare_insert_non_soo(const K& key) {
|
3806
|
+
assert(!is_soo());
|
3087
3807
|
prefetch_heap_block();
|
3088
3808
|
auto hash = hash_ref()(key);
|
3089
3809
|
auto seq = probe(common(), hash);
|
@@ -3094,65 +3814,92 @@ class raw_hash_set {
|
|
3094
3814
|
if (ABSL_PREDICT_TRUE(PolicyTraits::apply(
|
3095
3815
|
EqualElement<K>{key, eq_ref()},
|
3096
3816
|
PolicyTraits::element(slot_array() + seq.offset(i)))))
|
3097
|
-
return {seq.offset(i), false};
|
3817
|
+
return {iterator_at(seq.offset(i)), false};
|
3818
|
+
}
|
3819
|
+
auto mask_empty = g.MaskEmpty();
|
3820
|
+
if (ABSL_PREDICT_TRUE(mask_empty)) {
|
3821
|
+
size_t target = seq.offset(
|
3822
|
+
GetInsertionOffset(mask_empty, capacity(), hash, control()));
|
3823
|
+
return {iterator_at(PrepareInsertNonSoo(common(), hash,
|
3824
|
+
FindInfo{target, seq.index()},
|
3825
|
+
GetPolicyFunctions())),
|
3826
|
+
true};
|
3098
3827
|
}
|
3099
|
-
if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break;
|
3100
3828
|
seq.next();
|
3101
3829
|
assert(seq.index() <= capacity() && "full table!");
|
3102
3830
|
}
|
3103
|
-
return {prepare_insert(hash), true};
|
3104
3831
|
}
|
3105
3832
|
|
3106
|
-
|
3107
|
-
//
|
3108
|
-
//
|
3109
|
-
|
3110
|
-
|
3111
|
-
|
3112
|
-
|
3113
|
-
|
3114
|
-
|
3115
|
-
|
3116
|
-
|
3117
|
-
|
3118
|
-
|
3119
|
-
|
3120
|
-
|
3121
|
-
|
3122
|
-
|
3123
|
-
|
3124
|
-
|
3125
|
-
|
3126
|
-
|
3127
|
-
|
3128
|
-
|
3129
|
-
|
3130
|
-
|
3131
|
-
|
3132
|
-
|
3133
|
-
|
3134
|
-
|
3135
|
-
|
3136
|
-
|
3137
|
-
|
3138
|
-
|
3139
|
-
|
3833
|
+
protected:
|
3834
|
+
// Asserts that hash and equal functors provided by the user are consistent,
|
3835
|
+
// meaning that `eq(k1, k2)` implies `hash(k1)==hash(k2)`.
|
3836
|
+
template <class K>
|
3837
|
+
void AssertHashEqConsistent(ABSL_ATTRIBUTE_UNUSED const K& key) {
|
3838
|
+
#ifndef NDEBUG
|
3839
|
+
if (empty()) return;
|
3840
|
+
|
3841
|
+
const size_t hash_of_arg = hash_ref()(key);
|
3842
|
+
const auto assert_consistent = [&](const ctrl_t*, slot_type* slot) {
|
3843
|
+
const value_type& element = PolicyTraits::element(slot);
|
3844
|
+
const bool is_key_equal =
|
3845
|
+
PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element);
|
3846
|
+
if (!is_key_equal) return;
|
3847
|
+
|
3848
|
+
const size_t hash_of_slot =
|
3849
|
+
PolicyTraits::apply(HashElement{hash_ref()}, element);
|
3850
|
+
const bool is_hash_equal = hash_of_arg == hash_of_slot;
|
3851
|
+
if (!is_hash_equal) {
|
3852
|
+
// In this case, we're going to crash. Do a couple of other checks for
|
3853
|
+
// idempotence issues. Recalculating hash/eq here is also convenient for
|
3854
|
+
// debugging with gdb/lldb.
|
3855
|
+
const size_t once_more_hash_arg = hash_ref()(key);
|
3856
|
+
assert(hash_of_arg == once_more_hash_arg && "hash is not idempotent.");
|
3857
|
+
const size_t once_more_hash_slot =
|
3858
|
+
PolicyTraits::apply(HashElement{hash_ref()}, element);
|
3859
|
+
assert(hash_of_slot == once_more_hash_slot &&
|
3860
|
+
"hash is not idempotent.");
|
3861
|
+
const bool once_more_eq =
|
3862
|
+
PolicyTraits::apply(EqualElement<K>{key, eq_ref()}, element);
|
3863
|
+
assert(is_key_equal == once_more_eq && "equality is not idempotent.");
|
3864
|
+
}
|
3865
|
+
assert((!is_key_equal || is_hash_equal) &&
|
3866
|
+
"eq(k1, k2) must imply that hash(k1) == hash(k2). "
|
3867
|
+
"hash/eq functors are inconsistent.");
|
3868
|
+
};
|
3869
|
+
|
3870
|
+
if (is_soo()) {
|
3871
|
+
assert_consistent(/*unused*/ nullptr, soo_slot());
|
3872
|
+
return;
|
3873
|
+
}
|
3874
|
+
// We only do validation for small tables so that it's constant time.
|
3875
|
+
if (capacity() > 16) return;
|
3876
|
+
IterateOverFullSlots(common(), slot_array(), assert_consistent);
|
3877
|
+
#endif
|
3878
|
+
}
|
3879
|
+
|
3880
|
+
// Attempts to find `key` in the table; if it isn't found, returns an iterator
|
3881
|
+
// where the value can be inserted into, with the control byte already set to
|
3882
|
+
// `key`'s H2. Returns a bool indicating whether an insertion can take place.
|
3883
|
+
template <class K>
|
3884
|
+
std::pair<iterator, bool> find_or_prepare_insert(const K& key) {
|
3885
|
+
AssertHashEqConsistent(key);
|
3886
|
+
if (is_soo()) return find_or_prepare_insert_soo(key);
|
3887
|
+
return find_or_prepare_insert_non_soo(key);
|
3140
3888
|
}
|
3141
3889
|
|
3142
3890
|
// Constructs the value in the space pointed by the iterator. This only works
|
3143
3891
|
// after an unsuccessful find_or_prepare_insert() and before any other
|
3144
3892
|
// modifications happen in the raw_hash_set.
|
3145
3893
|
//
|
3146
|
-
// PRECONDITION:
|
3147
|
-
//
|
3148
|
-
//
|
3894
|
+
// PRECONDITION: iter was returned from find_or_prepare_insert(k), where k is
|
3895
|
+
// the key decomposed from `forward<Args>(args)...`, and the bool returned by
|
3896
|
+
// find_or_prepare_insert(k) was true.
|
3149
3897
|
// POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...).
|
3150
3898
|
template <class... Args>
|
3151
|
-
void emplace_at(
|
3152
|
-
construct(
|
3899
|
+
void emplace_at(iterator iter, Args&&... args) {
|
3900
|
+
construct(iter.slot(), std::forward<Args>(args)...);
|
3153
3901
|
|
3154
|
-
assert(PolicyTraits::apply(FindElement{*this}, *
|
3155
|
-
iterator_at(i) &&
|
3902
|
+
assert(PolicyTraits::apply(FindElement{*this}, *iter) == iter &&
|
3156
3903
|
"constructed value does not match the lookup key");
|
3157
3904
|
}
|
3158
3905
|
|
@@ -3160,7 +3907,7 @@ class raw_hash_set {
|
|
3160
3907
|
return {control() + i, slot_array() + i, common().generation_ptr()};
|
3161
3908
|
}
|
3162
3909
|
const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND {
|
3163
|
-
return
|
3910
|
+
return const_cast<raw_hash_set*>(this)->iterator_at(i);
|
3164
3911
|
}
|
3165
3912
|
|
3166
3913
|
reference unchecked_deref(iterator it) { return it.unchecked_deref(); }
|
@@ -3178,13 +3925,25 @@ class raw_hash_set {
|
|
3178
3925
|
// side-effect.
|
3179
3926
|
//
|
3180
3927
|
// See `CapacityToGrowth()`.
|
3181
|
-
size_t growth_left() const {
|
3182
|
-
|
3928
|
+
size_t growth_left() const {
|
3929
|
+
assert(!is_soo());
|
3930
|
+
return common().growth_left();
|
3931
|
+
}
|
3932
|
+
|
3933
|
+
GrowthInfo& growth_info() {
|
3934
|
+
assert(!is_soo());
|
3935
|
+
return common().growth_info();
|
3936
|
+
}
|
3937
|
+
GrowthInfo growth_info() const {
|
3938
|
+
assert(!is_soo());
|
3939
|
+
return common().growth_info();
|
3940
|
+
}
|
3183
3941
|
|
3184
3942
|
// Prefetch the heap-allocated memory region to resolve potential TLB and
|
3185
3943
|
// cache misses. This is intended to overlap with execution of calculating the
|
3186
3944
|
// hash for a key.
|
3187
3945
|
void prefetch_heap_block() const {
|
3946
|
+
assert(!is_soo());
|
3188
3947
|
#if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__)
|
3189
3948
|
__builtin_prefetch(control(), 0, 1);
|
3190
3949
|
#endif
|
@@ -3193,11 +3952,31 @@ class raw_hash_set {
|
|
3193
3952
|
CommonFields& common() { return settings_.template get<0>(); }
|
3194
3953
|
const CommonFields& common() const { return settings_.template get<0>(); }
|
3195
3954
|
|
3196
|
-
ctrl_t* control() const {
|
3955
|
+
ctrl_t* control() const {
|
3956
|
+
assert(!is_soo());
|
3957
|
+
return common().control();
|
3958
|
+
}
|
3197
3959
|
slot_type* slot_array() const {
|
3960
|
+
assert(!is_soo());
|
3198
3961
|
return static_cast<slot_type*>(common().slot_array());
|
3199
3962
|
}
|
3200
|
-
|
3963
|
+
slot_type* soo_slot() {
|
3964
|
+
assert(is_soo());
|
3965
|
+
return static_cast<slot_type*>(common().soo_data());
|
3966
|
+
}
|
3967
|
+
const slot_type* soo_slot() const {
|
3968
|
+
return const_cast<raw_hash_set*>(this)->soo_slot();
|
3969
|
+
}
|
3970
|
+
iterator soo_iterator() {
|
3971
|
+
return {SooControl(), soo_slot(), common().generation_ptr()};
|
3972
|
+
}
|
3973
|
+
const_iterator soo_iterator() const {
|
3974
|
+
return const_cast<raw_hash_set*>(this)->soo_iterator();
|
3975
|
+
}
|
3976
|
+
HashtablezInfoHandle infoz() {
|
3977
|
+
assert(!is_soo());
|
3978
|
+
return common().infoz();
|
3979
|
+
}
|
3201
3980
|
|
3202
3981
|
hasher& hash_ref() { return settings_.template get<1>(); }
|
3203
3982
|
const hasher& hash_ref() const { return settings_.template get<1>(); }
|
@@ -3208,12 +3987,9 @@ class raw_hash_set {
|
|
3208
3987
|
return settings_.template get<3>();
|
3209
3988
|
}
|
3210
3989
|
|
3211
|
-
|
3212
|
-
|
3213
|
-
|
3214
|
-
return PolicyTraits::apply(
|
3215
|
-
HashElement{h->hash_ref()},
|
3216
|
-
PolicyTraits::element(static_cast<slot_type*>(slot)));
|
3990
|
+
static const void* get_hash_ref_fn(const CommonFields& common) {
|
3991
|
+
auto* h = reinterpret_cast<const raw_hash_set*>(&common);
|
3992
|
+
return &h->hash_ref();
|
3217
3993
|
}
|
3218
3994
|
static void transfer_slot_fn(void* set, void* dst, void* src) {
|
3219
3995
|
auto* h = static_cast<raw_hash_set*>(set);
|
@@ -3236,13 +4012,18 @@ class raw_hash_set {
|
|
3236
4012
|
static const PolicyFunctions& GetPolicyFunctions() {
|
3237
4013
|
static constexpr PolicyFunctions value = {
|
3238
4014
|
sizeof(slot_type),
|
3239
|
-
|
4015
|
+
// TODO(b/328722020): try to type erase
|
4016
|
+
// for standard layout and alignof(Hash) <= alignof(CommonFields).
|
4017
|
+
std::is_empty<hasher>::value ? &GetHashRefForEmptyHasher
|
4018
|
+
: &raw_hash_set::get_hash_ref_fn,
|
4019
|
+
PolicyTraits::template get_hash_slot_fn<hasher>(),
|
3240
4020
|
PolicyTraits::transfer_uses_memcpy()
|
3241
4021
|
? TransferRelocatable<sizeof(slot_type)>
|
3242
4022
|
: &raw_hash_set::transfer_slot_fn,
|
3243
4023
|
(std::is_same<SlotAlloc, std::allocator<slot_type>>::value
|
3244
4024
|
? &DeallocateStandard<alignof(slot_type)>
|
3245
4025
|
: &raw_hash_set::dealloc_fn),
|
4026
|
+
&raw_hash_set::resize_impl,
|
3246
4027
|
};
|
3247
4028
|
return value;
|
3248
4029
|
}
|
@@ -3252,22 +4033,78 @@ class raw_hash_set {
|
|
3252
4033
|
// fields that occur after CommonFields.
|
3253
4034
|
absl::container_internal::CompressedTuple<CommonFields, hasher, key_equal,
|
3254
4035
|
allocator_type>
|
3255
|
-
settings_{CommonFields
|
4036
|
+
settings_{CommonFields::CreateDefault<SooEnabled()>(), hasher{},
|
4037
|
+
key_equal{}, allocator_type{}};
|
4038
|
+
};
|
4039
|
+
|
4040
|
+
// Friend access for free functions in raw_hash_set.h.
|
4041
|
+
struct HashtableFreeFunctionsAccess {
|
4042
|
+
template <class Predicate, typename Set>
|
4043
|
+
static typename Set::size_type EraseIf(Predicate& pred, Set* c) {
|
4044
|
+
if (c->empty()) {
|
4045
|
+
return 0;
|
4046
|
+
}
|
4047
|
+
if (c->is_soo()) {
|
4048
|
+
auto it = c->soo_iterator();
|
4049
|
+
if (!pred(*it)) {
|
4050
|
+
assert(c->size() == 1 && "hash table was modified unexpectedly");
|
4051
|
+
return 0;
|
4052
|
+
}
|
4053
|
+
c->destroy(it.slot());
|
4054
|
+
c->common().set_empty_soo();
|
4055
|
+
return 1;
|
4056
|
+
}
|
4057
|
+
ABSL_ATTRIBUTE_UNUSED const size_t original_size_for_assert = c->size();
|
4058
|
+
size_t num_deleted = 0;
|
4059
|
+
IterateOverFullSlots(
|
4060
|
+
c->common(), c->slot_array(), [&](const ctrl_t* ctrl, auto* slot) {
|
4061
|
+
if (pred(Set::PolicyTraits::element(slot))) {
|
4062
|
+
c->destroy(slot);
|
4063
|
+
EraseMetaOnly(c->common(), static_cast<size_t>(ctrl - c->control()),
|
4064
|
+
sizeof(*slot));
|
4065
|
+
++num_deleted;
|
4066
|
+
}
|
4067
|
+
});
|
4068
|
+
// NOTE: IterateOverFullSlots allow removal of the current element, so we
|
4069
|
+
// verify the size additionally here.
|
4070
|
+
assert(original_size_for_assert - num_deleted == c->size() &&
|
4071
|
+
"hash table was modified unexpectedly");
|
4072
|
+
return num_deleted;
|
4073
|
+
}
|
4074
|
+
|
4075
|
+
template <class Callback, typename Set>
|
4076
|
+
static void ForEach(Callback& cb, Set* c) {
|
4077
|
+
if (c->empty()) {
|
4078
|
+
return;
|
4079
|
+
}
|
4080
|
+
if (c->is_soo()) {
|
4081
|
+
cb(*c->soo_iterator());
|
4082
|
+
return;
|
4083
|
+
}
|
4084
|
+
using ElementTypeWithConstness = decltype(*c->begin());
|
4085
|
+
IterateOverFullSlots(
|
4086
|
+
c->common(), c->slot_array(), [&cb](const ctrl_t*, auto* slot) {
|
4087
|
+
ElementTypeWithConstness& element = Set::PolicyTraits::element(slot);
|
4088
|
+
cb(element);
|
4089
|
+
});
|
4090
|
+
}
|
3256
4091
|
};
|
3257
4092
|
|
3258
4093
|
// Erases all elements that satisfy the predicate `pred` from the container `c`.
|
3259
4094
|
template <typename P, typename H, typename E, typename A, typename Predicate>
|
3260
4095
|
typename raw_hash_set<P, H, E, A>::size_type EraseIf(
|
3261
4096
|
Predicate& pred, raw_hash_set<P, H, E, A>* c) {
|
3262
|
-
|
3263
|
-
|
3264
|
-
|
3265
|
-
|
3266
|
-
|
3267
|
-
|
3268
|
-
|
3269
|
-
|
3270
|
-
|
4097
|
+
return HashtableFreeFunctionsAccess::EraseIf(pred, c);
|
4098
|
+
}
|
4099
|
+
|
4100
|
+
// Calls `cb` for all elements in the container `c`.
|
4101
|
+
template <typename P, typename H, typename E, typename A, typename Callback>
|
4102
|
+
void ForEach(Callback& cb, raw_hash_set<P, H, E, A>* c) {
|
4103
|
+
return HashtableFreeFunctionsAccess::ForEach(cb, c);
|
4104
|
+
}
|
4105
|
+
template <typename P, typename H, typename E, typename A, typename Callback>
|
4106
|
+
void ForEach(Callback& cb, const raw_hash_set<P, H, E, A>* c) {
|
4107
|
+
return HashtableFreeFunctionsAccess::ForEach(cb, c);
|
3271
4108
|
}
|
3272
4109
|
|
3273
4110
|
namespace hashtable_debug_internal {
|
@@ -3278,6 +4115,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> {
|
|
3278
4115
|
|
3279
4116
|
static size_t GetNumProbes(const Set& set,
|
3280
4117
|
const typename Set::key_type& key) {
|
4118
|
+
if (set.is_soo()) return 0;
|
3281
4119
|
size_t num_probes = 0;
|
3282
4120
|
size_t hash = set.hash_ref()(key);
|
3283
4121
|
auto seq = probe(set.common(), hash);
|
@@ -3301,7 +4139,8 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> {
|
|
3301
4139
|
static size_t AllocatedByteSize(const Set& c) {
|
3302
4140
|
size_t capacity = c.capacity();
|
3303
4141
|
if (capacity == 0) return 0;
|
3304
|
-
size_t m =
|
4142
|
+
size_t m =
|
4143
|
+
c.is_soo() ? 0 : c.common().alloc_size(sizeof(Slot), alignof(Slot));
|
3305
4144
|
|
3306
4145
|
size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr));
|
3307
4146
|
if (per_slot != ~size_t{}) {
|
@@ -3321,5 +4160,7 @@ ABSL_NAMESPACE_END
|
|
3321
4160
|
} // namespace absl
|
3322
4161
|
|
3323
4162
|
#undef ABSL_SWISSTABLE_ENABLE_GENERATIONS
|
4163
|
+
#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED
|
4164
|
+
#undef ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN
|
3324
4165
|
|
3325
4166
|
#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_
|