grpc 1.17.1 → 1.18.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of grpc might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Makefile +1228 -988
- data/etc/roots.pem +242 -30
- data/include/grpc/grpc.h +2 -1
- data/include/grpc/grpc_security_constants.h +3 -3
- data/include/grpc/impl/codegen/atm_gcc_sync.h +2 -0
- data/include/grpc/impl/codegen/atm_windows.h +2 -0
- data/include/grpc/impl/codegen/compression_types.h +2 -1
- data/include/grpc/impl/codegen/grpc_types.h +1 -1
- data/include/grpc/impl/codegen/port_platform.h +9 -0
- data/src/core/ext/filters/client_channel/client_channel.cc +163 -882
- data/src/core/ext/filters/client_channel/health/health_check_client.cc +2 -4
- data/src/core/ext/filters/client_channel/health/health_check_client.h +2 -3
- data/src/core/ext/filters/client_channel/lb_policy.cc +1 -1
- data/src/core/ext/filters/client_channel/lb_policy.h +8 -17
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +176 -216
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h +1 -1
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc +20 -23
- data/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h +1 -1
- data/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +49 -52
- data/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +13 -35
- data/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +31 -30
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +69 -225
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h +1 -1
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc +20 -23
- data/src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h +1 -1
- data/src/core/ext/filters/client_channel/lb_policy_factory.h +2 -84
- data/src/core/ext/filters/client_channel/request_routing.cc +936 -0
- data/src/core/ext/filters/client_channel/request_routing.h +177 -0
- data/src/core/ext/filters/client_channel/resolver.cc +1 -1
- data/src/core/ext/filters/client_channel/resolver.h +1 -1
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +37 -26
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +30 -18
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +119 -100
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +8 -5
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc +5 -4
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc +2 -1
- data/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc +12 -14
- data/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +5 -9
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +2 -1
- data/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h +1 -2
- data/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc +17 -17
- data/src/core/ext/filters/client_channel/resolver_result_parsing.cc +45 -52
- data/src/core/ext/filters/client_channel/resolver_result_parsing.h +13 -17
- data/src/core/ext/filters/client_channel/server_address.cc +103 -0
- data/src/core/ext/filters/client_channel/server_address.h +108 -0
- data/src/core/ext/filters/client_channel/subchannel.cc +10 -8
- data/src/core/ext/filters/client_channel/subchannel.h +9 -6
- data/src/core/ext/filters/client_channel/subchannel_index.cc +20 -27
- data/src/core/ext/transport/chttp2/client/chttp2_connector.cc +3 -2
- data/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc +8 -9
- data/src/core/ext/transport/chttp2/server/chttp2_server.cc +1 -1
- data/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc +1 -1
- data/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc +8 -11
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +24 -54
- data/src/core/ext/transport/chttp2/transport/chttp2_transport.h +3 -1
- data/src/core/ext/transport/chttp2/transport/context_list.cc +67 -0
- data/src/core/ext/transport/chttp2/transport/context_list.h +53 -0
- data/src/core/ext/transport/chttp2/transport/internal.h +38 -11
- data/src/core/ext/transport/chttp2/transport/writing.cc +5 -0
- data/src/core/ext/transport/inproc/inproc_transport.cc +1 -1
- data/src/core/lib/channel/channelz.cc +19 -18
- data/src/core/lib/channel/channelz.h +7 -1
- data/src/core/lib/channel/channelz_registry.cc +3 -2
- data/src/core/lib/debug/trace.cc +3 -0
- data/src/core/lib/debug/trace.h +5 -3
- data/src/core/lib/gpr/sync_posix.cc +96 -4
- data/src/core/lib/gprpp/inlined_vector.h +25 -19
- data/src/core/lib/gprpp/memory.h +2 -11
- data/src/core/lib/gprpp/orphanable.h +18 -82
- data/src/core/lib/gprpp/ref_counted.h +75 -84
- data/src/core/lib/gprpp/ref_counted_ptr.h +22 -17
- data/src/core/lib/http/httpcli_security_connector.cc +101 -94
- data/src/core/lib/http/parser.h +5 -5
- data/src/core/lib/iomgr/buffer_list.cc +16 -5
- data/src/core/lib/iomgr/buffer_list.h +10 -3
- data/src/core/lib/iomgr/call_combiner.cc +50 -2
- data/src/core/lib/iomgr/call_combiner.h +29 -2
- data/src/core/lib/iomgr/dynamic_annotations.h +67 -0
- data/src/core/lib/iomgr/endpoint.cc +4 -0
- data/src/core/lib/iomgr/endpoint.h +3 -0
- data/src/core/lib/iomgr/endpoint_pair_posix.cc +2 -2
- data/src/core/lib/iomgr/ev_epoll1_linux.cc +4 -0
- data/src/core/lib/iomgr/ev_epollex_linux.cc +4 -0
- data/src/core/lib/iomgr/ev_poll_posix.cc +4 -0
- data/src/core/lib/iomgr/ev_posix.cc +15 -7
- data/src/core/lib/iomgr/ev_posix.h +10 -0
- data/src/core/lib/iomgr/exec_ctx.cc +13 -0
- data/src/core/lib/iomgr/fork_posix.cc +1 -1
- data/src/core/lib/iomgr/internal_errqueue.cc +36 -3
- data/src/core/lib/iomgr/internal_errqueue.h +7 -1
- data/src/core/lib/iomgr/iomgr.cc +7 -0
- data/src/core/lib/iomgr/iomgr.h +4 -0
- data/src/core/lib/iomgr/iomgr_custom.cc +3 -1
- data/src/core/lib/iomgr/iomgr_internal.cc +4 -0
- data/src/core/lib/iomgr/iomgr_internal.h +4 -0
- data/src/core/lib/iomgr/iomgr_posix.cc +6 -1
- data/src/core/lib/iomgr/iomgr_windows.cc +4 -1
- data/src/core/lib/iomgr/port.h +1 -2
- data/src/core/lib/iomgr/resource_quota.cc +1 -0
- data/src/core/lib/iomgr/sockaddr_utils.cc +1 -0
- data/src/core/lib/iomgr/tcp_custom.cc +4 -1
- data/src/core/lib/iomgr/tcp_posix.cc +95 -35
- data/src/core/lib/iomgr/tcp_windows.cc +4 -1
- data/src/core/lib/iomgr/timer_manager.cc +6 -0
- data/src/core/lib/security/context/security_context.cc +75 -108
- data/src/core/lib/security/context/security_context.h +59 -35
- data/src/core/lib/security/credentials/alts/alts_credentials.cc +36 -48
- data/src/core/lib/security/credentials/alts/alts_credentials.h +37 -10
- data/src/core/lib/security/credentials/composite/composite_credentials.cc +97 -157
- data/src/core/lib/security/credentials/composite/composite_credentials.h +60 -24
- data/src/core/lib/security/credentials/credentials.cc +18 -142
- data/src/core/lib/security/credentials/credentials.h +119 -95
- data/src/core/lib/security/credentials/fake/fake_credentials.cc +46 -71
- data/src/core/lib/security/credentials/fake/fake_credentials.h +23 -5
- data/src/core/lib/security/credentials/google_default/google_default_credentials.cc +144 -51
- data/src/core/lib/security/credentials/google_default/google_default_credentials.h +28 -5
- data/src/core/lib/security/credentials/iam/iam_credentials.cc +27 -35
- data/src/core/lib/security/credentials/iam/iam_credentials.h +18 -4
- data/src/core/lib/security/credentials/jwt/jwt_credentials.cc +60 -69
- data/src/core/lib/security/credentials/jwt/jwt_credentials.h +29 -10
- data/src/core/lib/security/credentials/jwt/jwt_verifier.cc +2 -0
- data/src/core/lib/security/credentials/local/local_credentials.cc +19 -32
- data/src/core/lib/security/credentials/local/local_credentials.h +32 -11
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +130 -149
- data/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +74 -29
- data/src/core/lib/security/credentials/plugin/plugin_credentials.cc +59 -77
- data/src/core/lib/security/credentials/plugin/plugin_credentials.h +40 -17
- data/src/core/lib/security/credentials/ssl/ssl_credentials.cc +66 -83
- data/src/core/lib/security/credentials/ssl/ssl_credentials.h +58 -15
- data/src/core/lib/security/security_connector/alts/alts_security_connector.cc +152 -177
- data/src/core/lib/security/security_connector/alts/alts_security_connector.h +12 -10
- data/src/core/lib/security/security_connector/fake/fake_security_connector.cc +210 -215
- data/src/core/lib/security/security_connector/fake/fake_security_connector.h +9 -6
- data/src/core/lib/security/security_connector/local/local_security_connector.cc +176 -169
- data/src/core/lib/security/security_connector/local/local_security_connector.h +10 -9
- data/src/core/lib/security/security_connector/security_connector.cc +41 -124
- data/src/core/lib/security/security_connector/security_connector.h +102 -105
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc +348 -370
- data/src/core/lib/security/security_connector/ssl/ssl_security_connector.h +14 -12
- data/src/core/lib/security/security_connector/ssl_utils.cc +13 -9
- data/src/core/lib/security/security_connector/ssl_utils.h +3 -1
- data/src/core/lib/security/transport/client_auth_filter.cc +50 -50
- data/src/core/lib/security/transport/secure_endpoint.cc +7 -1
- data/src/core/lib/security/transport/security_handshaker.cc +82 -66
- data/src/core/lib/security/transport/server_auth_filter.cc +15 -13
- data/src/core/lib/surface/init.cc +1 -0
- data/src/core/lib/surface/server.cc +13 -11
- data/src/core/lib/surface/server.h +6 -6
- data/src/core/lib/surface/version.cc +2 -2
- data/src/core/lib/transport/metadata.cc +1 -0
- data/src/core/lib/transport/static_metadata.cc +228 -221
- data/src/core/lib/transport/static_metadata.h +75 -71
- data/src/core/lib/transport/transport.cc +2 -1
- data/src/core/lib/transport/transport.h +5 -1
- data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +9 -2
- data/src/core/tsi/ssl_transport_security.cc +35 -24
- data/src/ruby/ext/grpc/rb_grpc_imports.generated.h +1 -1
- data/src/ruby/lib/grpc/generic/rpc_server.rb +61 -0
- data/src/ruby/lib/grpc/generic/service.rb +1 -1
- data/src/ruby/lib/grpc/version.rb +1 -1
- data/src/ruby/pb/grpc/health/checker.rb +2 -3
- data/src/ruby/spec/generic/rpc_server_spec.rb +22 -0
- data/src/ruby/spec/support/services.rb +1 -0
- metadata +37 -32
- data/src/core/ext/filters/client_channel/lb_policy_factory.cc +0 -163
@@ -21,7 +21,7 @@
|
|
21
21
|
|
22
22
|
#include <grpc/support/port_platform.h>
|
23
23
|
|
24
|
-
#include
|
24
|
+
#include <grpc/impl/codegen/grpc_types.h>
|
25
25
|
|
26
26
|
/// Makes any necessary modifications to \a args for use in the grpclb
|
27
27
|
/// balancer channel.
|
@@ -26,6 +26,7 @@
|
|
26
26
|
#include <grpc/support/string_util.h>
|
27
27
|
|
28
28
|
#include "src/core/ext/filters/client_channel/client_channel.h"
|
29
|
+
#include "src/core/ext/filters/client_channel/server_address.h"
|
29
30
|
#include "src/core/lib/channel/channel_args.h"
|
30
31
|
#include "src/core/lib/gpr/string.h"
|
31
32
|
#include "src/core/lib/iomgr/sockaddr_utils.h"
|
@@ -42,22 +43,23 @@ int BalancerNameCmp(const grpc_core::UniquePtr<char>& a,
|
|
42
43
|
}
|
43
44
|
|
44
45
|
RefCountedPtr<TargetAuthorityTable> CreateTargetAuthorityTable(
|
45
|
-
|
46
|
+
const ServerAddressList& addresses) {
|
46
47
|
TargetAuthorityTable::Entry* target_authority_entries =
|
47
|
-
static_cast<TargetAuthorityTable::Entry*>(
|
48
|
-
sizeof(*target_authority_entries) * addresses
|
49
|
-
for (size_t i = 0; i < addresses
|
48
|
+
static_cast<TargetAuthorityTable::Entry*>(
|
49
|
+
gpr_zalloc(sizeof(*target_authority_entries) * addresses.size()));
|
50
|
+
for (size_t i = 0; i < addresses.size(); ++i) {
|
50
51
|
char* addr_str;
|
51
|
-
GPR_ASSERT(
|
52
|
-
|
52
|
+
GPR_ASSERT(
|
53
|
+
grpc_sockaddr_to_string(&addr_str, &addresses[i].address(), true) > 0);
|
53
54
|
target_authority_entries[i].key = grpc_slice_from_copied_string(addr_str);
|
54
|
-
target_authority_entries[i].value.reset(
|
55
|
-
gpr_strdup(addresses->addresses[i].balancer_name));
|
56
55
|
gpr_free(addr_str);
|
56
|
+
char* balancer_name = grpc_channel_arg_get_string(grpc_channel_args_find(
|
57
|
+
addresses[i].args(), GRPC_ARG_ADDRESS_BALANCER_NAME));
|
58
|
+
target_authority_entries[i].value.reset(gpr_strdup(balancer_name));
|
57
59
|
}
|
58
60
|
RefCountedPtr<TargetAuthorityTable> target_authority_table =
|
59
|
-
TargetAuthorityTable::Create(addresses
|
60
|
-
|
61
|
+
TargetAuthorityTable::Create(addresses.size(), target_authority_entries,
|
62
|
+
BalancerNameCmp);
|
61
63
|
gpr_free(target_authority_entries);
|
62
64
|
return target_authority_table;
|
63
65
|
}
|
@@ -72,13 +74,12 @@ grpc_channel_args* grpc_lb_policy_grpclb_modify_lb_channel_args(
|
|
72
74
|
grpc_arg args_to_add[2];
|
73
75
|
size_t num_args_to_add = 0;
|
74
76
|
// Add arg for targets info table.
|
75
|
-
|
76
|
-
|
77
|
-
GPR_ASSERT(
|
78
|
-
grpc_lb_addresses* addresses =
|
79
|
-
static_cast<grpc_lb_addresses*>(arg->value.pointer.p);
|
77
|
+
grpc_core::ServerAddressList* addresses =
|
78
|
+
grpc_core::FindServerAddressListChannelArg(args);
|
79
|
+
GPR_ASSERT(addresses != nullptr);
|
80
80
|
grpc_core::RefCountedPtr<grpc_core::TargetAuthorityTable>
|
81
|
-
target_authority_table =
|
81
|
+
target_authority_table =
|
82
|
+
grpc_core::CreateTargetAuthorityTable(*addresses);
|
82
83
|
args_to_add[num_args_to_add++] =
|
83
84
|
grpc_core::CreateTargetAuthorityTableChannelArg(
|
84
85
|
target_authority_table.get());
|
@@ -87,22 +88,18 @@ grpc_channel_args* grpc_lb_policy_grpclb_modify_lb_channel_args(
|
|
87
88
|
// bearer token credentials.
|
88
89
|
grpc_channel_credentials* channel_credentials =
|
89
90
|
grpc_channel_credentials_find_in_args(args);
|
90
|
-
grpc_channel_credentials
|
91
|
+
grpc_core::RefCountedPtr<grpc_channel_credentials> creds_sans_call_creds;
|
91
92
|
if (channel_credentials != nullptr) {
|
92
93
|
creds_sans_call_creds =
|
93
|
-
|
94
|
-
channel_credentials);
|
94
|
+
channel_credentials->duplicate_without_call_credentials();
|
95
95
|
GPR_ASSERT(creds_sans_call_creds != nullptr);
|
96
96
|
args_to_remove[num_args_to_remove++] = GRPC_ARG_CHANNEL_CREDENTIALS;
|
97
97
|
args_to_add[num_args_to_add++] =
|
98
|
-
grpc_channel_credentials_to_arg(creds_sans_call_creds);
|
98
|
+
grpc_channel_credentials_to_arg(creds_sans_call_creds.get());
|
99
99
|
}
|
100
100
|
grpc_channel_args* result = grpc_channel_args_copy_and_add_and_remove(
|
101
101
|
args, args_to_remove, num_args_to_remove, args_to_add, num_args_to_add);
|
102
102
|
// Clean up.
|
103
103
|
grpc_channel_args_destroy(args);
|
104
|
-
if (creds_sans_call_creds != nullptr) {
|
105
|
-
grpc_channel_credentials_unref(creds_sans_call_creds);
|
106
|
-
}
|
107
104
|
return result;
|
108
105
|
}
|
@@ -25,7 +25,7 @@
|
|
25
25
|
|
26
26
|
#include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h"
|
27
27
|
#include "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
|
28
|
-
#include "src/core/
|
28
|
+
#include "src/core/lib/iomgr/exec_ctx.h"
|
29
29
|
|
30
30
|
#define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128
|
31
31
|
|
@@ -24,6 +24,7 @@
|
|
24
24
|
|
25
25
|
#include "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h"
|
26
26
|
#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
|
27
|
+
#include "src/core/ext/filters/client_channel/server_address.h"
|
27
28
|
#include "src/core/ext/filters/client_channel/subchannel.h"
|
28
29
|
#include "src/core/ext/filters/client_channel/subchannel_index.h"
|
29
30
|
#include "src/core/lib/channel/channel_args.h"
|
@@ -42,10 +43,14 @@ namespace {
|
|
42
43
|
// pick_first LB policy
|
43
44
|
//
|
44
45
|
|
46
|
+
constexpr char kPickFirst[] = "pick_first";
|
47
|
+
|
45
48
|
class PickFirst : public LoadBalancingPolicy {
|
46
49
|
public:
|
47
50
|
explicit PickFirst(const Args& args);
|
48
51
|
|
52
|
+
const char* name() const override { return kPickFirst; }
|
53
|
+
|
49
54
|
void UpdateLocked(const grpc_channel_args& args,
|
50
55
|
grpc_json* lb_config) override;
|
51
56
|
bool PickLocked(PickState* pick, grpc_error** error) override;
|
@@ -75,11 +80,9 @@ class PickFirst : public LoadBalancingPolicy {
|
|
75
80
|
PickFirstSubchannelData(
|
76
81
|
SubchannelList<PickFirstSubchannelList, PickFirstSubchannelData>*
|
77
82
|
subchannel_list,
|
78
|
-
const
|
79
|
-
const grpc_lb_address& address, grpc_subchannel* subchannel,
|
83
|
+
const ServerAddress& address, grpc_subchannel* subchannel,
|
80
84
|
grpc_combiner* combiner)
|
81
|
-
: SubchannelData(subchannel_list,
|
82
|
-
combiner) {}
|
85
|
+
: SubchannelData(subchannel_list, address, subchannel, combiner) {}
|
83
86
|
|
84
87
|
void ProcessConnectivityChangeLocked(
|
85
88
|
grpc_connectivity_state connectivity_state, grpc_error* error) override;
|
@@ -95,7 +98,7 @@ class PickFirst : public LoadBalancingPolicy {
|
|
95
98
|
PickFirstSubchannelData> {
|
96
99
|
public:
|
97
100
|
PickFirstSubchannelList(PickFirst* policy, TraceFlag* tracer,
|
98
|
-
const
|
101
|
+
const ServerAddressList& addresses,
|
99
102
|
grpc_combiner* combiner,
|
100
103
|
grpc_client_channel_factory* client_channel_factory,
|
101
104
|
const grpc_channel_args& args)
|
@@ -235,7 +238,7 @@ void PickFirst::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
|
|
235
238
|
pending_picks_ = nullptr;
|
236
239
|
while (pick != nullptr) {
|
237
240
|
PickState* next = pick->next;
|
238
|
-
if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
|
241
|
+
if ((*pick->initial_metadata_flags & initial_metadata_flags_mask) ==
|
239
242
|
initial_metadata_flags_eq) {
|
240
243
|
GRPC_CLOSURE_SCHED(pick->on_complete,
|
241
244
|
GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
|
@@ -337,8 +340,8 @@ void PickFirst::UpdateChildRefsLocked() {
|
|
337
340
|
void PickFirst::UpdateLocked(const grpc_channel_args& args,
|
338
341
|
grpc_json* lb_config) {
|
339
342
|
AutoChildRefsUpdater guard(this);
|
340
|
-
const
|
341
|
-
if (
|
343
|
+
const ServerAddressList* addresses = FindServerAddressListChannelArg(&args);
|
344
|
+
if (addresses == nullptr) {
|
342
345
|
if (subchannel_list_ == nullptr) {
|
343
346
|
// If we don't have a current subchannel list, go into TRANSIENT FAILURE.
|
344
347
|
grpc_connectivity_state_set(
|
@@ -354,19 +357,17 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
|
|
354
357
|
}
|
355
358
|
return;
|
356
359
|
}
|
357
|
-
const grpc_lb_addresses* addresses =
|
358
|
-
static_cast<const grpc_lb_addresses*>(arg->value.pointer.p);
|
359
360
|
if (grpc_lb_pick_first_trace.enabled()) {
|
360
361
|
gpr_log(GPR_INFO,
|
361
362
|
"Pick First %p received update with %" PRIuPTR " addresses", this,
|
362
|
-
addresses->
|
363
|
+
addresses->size());
|
363
364
|
}
|
364
365
|
grpc_arg new_arg = grpc_channel_arg_integer_create(
|
365
366
|
const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1);
|
366
367
|
grpc_channel_args* new_args =
|
367
368
|
grpc_channel_args_copy_and_add(&args, &new_arg, 1);
|
368
369
|
auto subchannel_list = MakeOrphanable<PickFirstSubchannelList>(
|
369
|
-
this, &grpc_lb_pick_first_trace, addresses, combiner(),
|
370
|
+
this, &grpc_lb_pick_first_trace, *addresses, combiner(),
|
370
371
|
client_channel_factory(), *new_args);
|
371
372
|
grpc_channel_args_destroy(new_args);
|
372
373
|
if (subchannel_list->num_subchannels() == 0) {
|
@@ -380,6 +381,31 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
|
|
380
381
|
selected_ = nullptr;
|
381
382
|
return;
|
382
383
|
}
|
384
|
+
// If one of the subchannels in the new list is already in state
|
385
|
+
// READY, then select it immediately. This can happen when the
|
386
|
+
// currently selected subchannel is also present in the update. It
|
387
|
+
// can also happen if one of the subchannels in the update is already
|
388
|
+
// in the subchannel index because it's in use by another channel.
|
389
|
+
for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
|
390
|
+
PickFirstSubchannelData* sd = subchannel_list->subchannel(i);
|
391
|
+
grpc_error* error = GRPC_ERROR_NONE;
|
392
|
+
grpc_connectivity_state state = sd->CheckConnectivityStateLocked(&error);
|
393
|
+
GRPC_ERROR_UNREF(error);
|
394
|
+
if (state == GRPC_CHANNEL_READY) {
|
395
|
+
subchannel_list_ = std::move(subchannel_list);
|
396
|
+
sd->ProcessUnselectedReadyLocked();
|
397
|
+
sd->StartConnectivityWatchLocked();
|
398
|
+
// If there was a previously pending update (which may or may
|
399
|
+
// not have contained the currently selected subchannel), drop
|
400
|
+
// it, so that it doesn't override what we've done here.
|
401
|
+
latest_pending_subchannel_list_.reset();
|
402
|
+
// Make sure that subsequent calls to ExitIdleLocked() don't cause
|
403
|
+
// us to start watching a subchannel other than the one we've
|
404
|
+
// selected.
|
405
|
+
started_picking_ = true;
|
406
|
+
return;
|
407
|
+
}
|
408
|
+
}
|
383
409
|
if (selected_ == nullptr) {
|
384
410
|
// We don't yet have a selected subchannel, so replace the current
|
385
411
|
// subchannel list immediately.
|
@@ -387,46 +413,14 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
|
|
387
413
|
// If we've started picking, start trying to connect to the first
|
388
414
|
// subchannel in the new list.
|
389
415
|
if (started_picking_) {
|
390
|
-
|
391
|
-
|
416
|
+
// Note: No need to use CheckConnectivityStateAndStartWatchingLocked()
|
417
|
+
// here, since we've already checked the initial connectivity
|
418
|
+
// state of all subchannels above.
|
419
|
+
subchannel_list_->subchannel(0)->StartConnectivityWatchLocked();
|
392
420
|
}
|
393
421
|
} else {
|
394
|
-
// We do have a selected subchannel
|
395
|
-
//
|
396
|
-
for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
|
397
|
-
PickFirstSubchannelData* sd = subchannel_list->subchannel(i);
|
398
|
-
if (sd->subchannel() == selected_->subchannel()) {
|
399
|
-
// The currently selected subchannel is in the update: we are done.
|
400
|
-
if (grpc_lb_pick_first_trace.enabled()) {
|
401
|
-
gpr_log(GPR_INFO,
|
402
|
-
"Pick First %p found already selected subchannel %p "
|
403
|
-
"at update index %" PRIuPTR " of %" PRIuPTR "; update done",
|
404
|
-
this, selected_->subchannel(), i,
|
405
|
-
subchannel_list->num_subchannels());
|
406
|
-
}
|
407
|
-
// Make sure it's in state READY. It might not be if we grabbed
|
408
|
-
// the combiner while a connectivity state notification
|
409
|
-
// informing us otherwise is pending.
|
410
|
-
// Note that CheckConnectivityStateLocked() also takes a ref to
|
411
|
-
// the connected subchannel.
|
412
|
-
grpc_error* error = GRPC_ERROR_NONE;
|
413
|
-
if (sd->CheckConnectivityStateLocked(&error) == GRPC_CHANNEL_READY) {
|
414
|
-
selected_ = sd;
|
415
|
-
subchannel_list_ = std::move(subchannel_list);
|
416
|
-
sd->StartConnectivityWatchLocked();
|
417
|
-
// If there was a previously pending update (which may or may
|
418
|
-
// not have contained the currently selected subchannel), drop
|
419
|
-
// it, so that it doesn't override what we've done here.
|
420
|
-
latest_pending_subchannel_list_.reset();
|
421
|
-
return;
|
422
|
-
}
|
423
|
-
GRPC_ERROR_UNREF(error);
|
424
|
-
}
|
425
|
-
}
|
426
|
-
// Not keeping the previous selected subchannel, so set the latest
|
427
|
-
// pending subchannel list to the new subchannel list. We will wait
|
428
|
-
// for it to report READY before swapping it into the current
|
429
|
-
// subchannel list.
|
422
|
+
// We do have a selected subchannel, so keep using it until one of
|
423
|
+
// the subchannels in the new list reports READY.
|
430
424
|
if (latest_pending_subchannel_list_ != nullptr) {
|
431
425
|
if (grpc_lb_pick_first_trace.enabled()) {
|
432
426
|
gpr_log(GPR_INFO,
|
@@ -440,8 +434,11 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
|
|
440
434
|
// If we've started picking, start trying to connect to the first
|
441
435
|
// subchannel in the new list.
|
442
436
|
if (started_picking_) {
|
437
|
+
// Note: No need to use CheckConnectivityStateAndStartWatchingLocked()
|
438
|
+
// here, since we've already checked the initial connectivity
|
439
|
+
// state of all subchannels above.
|
443
440
|
latest_pending_subchannel_list_->subchannel(0)
|
444
|
-
->
|
441
|
+
->StartConnectivityWatchLocked();
|
445
442
|
}
|
446
443
|
}
|
447
444
|
}
|
@@ -629,7 +626,7 @@ class PickFirstFactory : public LoadBalancingPolicyFactory {
|
|
629
626
|
return OrphanablePtr<LoadBalancingPolicy>(New<PickFirst>(args));
|
630
627
|
}
|
631
628
|
|
632
|
-
const char* name() const override { return
|
629
|
+
const char* name() const override { return kPickFirst; }
|
633
630
|
};
|
634
631
|
|
635
632
|
} // namespace
|
@@ -53,10 +53,14 @@ namespace {
|
|
53
53
|
// round_robin LB policy
|
54
54
|
//
|
55
55
|
|
56
|
+
constexpr char kRoundRobin[] = "round_robin";
|
57
|
+
|
56
58
|
class RoundRobin : public LoadBalancingPolicy {
|
57
59
|
public:
|
58
60
|
explicit RoundRobin(const Args& args);
|
59
61
|
|
62
|
+
const char* name() const override { return kRoundRobin; }
|
63
|
+
|
60
64
|
void UpdateLocked(const grpc_channel_args& args,
|
61
65
|
grpc_json* lb_config) override;
|
62
66
|
bool PickLocked(PickState* pick, grpc_error** error) override;
|
@@ -82,8 +86,6 @@ class RoundRobin : public LoadBalancingPolicy {
|
|
82
86
|
|
83
87
|
// Data for a particular subchannel in a subchannel list.
|
84
88
|
// This subclass adds the following functionality:
|
85
|
-
// - Tracks user_data associated with each address, which will be
|
86
|
-
// returned along with picks that select the subchannel.
|
87
89
|
// - Tracks the previous connectivity state of the subchannel, so that
|
88
90
|
// we know how many subchannels are in each state.
|
89
91
|
class RoundRobinSubchannelData
|
@@ -93,26 +95,9 @@ class RoundRobin : public LoadBalancingPolicy {
|
|
93
95
|
RoundRobinSubchannelData(
|
94
96
|
SubchannelList<RoundRobinSubchannelList, RoundRobinSubchannelData>*
|
95
97
|
subchannel_list,
|
96
|
-
const
|
97
|
-
const grpc_lb_address& address, grpc_subchannel* subchannel,
|
98
|
+
const ServerAddress& address, grpc_subchannel* subchannel,
|
98
99
|
grpc_combiner* combiner)
|
99
|
-
: SubchannelData(subchannel_list,
|
100
|
-
combiner),
|
101
|
-
user_data_vtable_(user_data_vtable),
|
102
|
-
user_data_(user_data_vtable_ != nullptr
|
103
|
-
? user_data_vtable_->copy(address.user_data)
|
104
|
-
: nullptr) {}
|
105
|
-
|
106
|
-
void UnrefSubchannelLocked(const char* reason) override {
|
107
|
-
SubchannelData::UnrefSubchannelLocked(reason);
|
108
|
-
if (user_data_ != nullptr) {
|
109
|
-
GPR_ASSERT(user_data_vtable_ != nullptr);
|
110
|
-
user_data_vtable_->destroy(user_data_);
|
111
|
-
user_data_ = nullptr;
|
112
|
-
}
|
113
|
-
}
|
114
|
-
|
115
|
-
void* user_data() const { return user_data_; }
|
100
|
+
: SubchannelData(subchannel_list, address, subchannel, combiner) {}
|
116
101
|
|
117
102
|
grpc_connectivity_state connectivity_state() const {
|
118
103
|
return last_connectivity_state_;
|
@@ -125,8 +110,6 @@ class RoundRobin : public LoadBalancingPolicy {
|
|
125
110
|
void ProcessConnectivityChangeLocked(
|
126
111
|
grpc_connectivity_state connectivity_state, grpc_error* error) override;
|
127
112
|
|
128
|
-
const grpc_lb_user_data_vtable* user_data_vtable_;
|
129
|
-
void* user_data_ = nullptr;
|
130
113
|
grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_IDLE;
|
131
114
|
};
|
132
115
|
|
@@ -137,7 +120,7 @@ class RoundRobin : public LoadBalancingPolicy {
|
|
137
120
|
public:
|
138
121
|
RoundRobinSubchannelList(
|
139
122
|
RoundRobin* policy, TraceFlag* tracer,
|
140
|
-
const
|
123
|
+
const ServerAddressList& addresses, grpc_combiner* combiner,
|
141
124
|
grpc_client_channel_factory* client_channel_factory,
|
142
125
|
const grpc_channel_args& args)
|
143
126
|
: SubchannelList(policy, tracer, addresses, combiner,
|
@@ -312,7 +295,7 @@ void RoundRobin::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
|
|
312
295
|
pending_picks_ = nullptr;
|
313
296
|
while (pick != nullptr) {
|
314
297
|
PickState* next = pick->next;
|
315
|
-
if ((pick->initial_metadata_flags & initial_metadata_flags_mask) ==
|
298
|
+
if ((*pick->initial_metadata_flags & initial_metadata_flags_mask) ==
|
316
299
|
initial_metadata_flags_eq) {
|
317
300
|
pick->connected_subchannel.reset();
|
318
301
|
GRPC_CLOSURE_SCHED(pick->on_complete,
|
@@ -354,9 +337,6 @@ bool RoundRobin::DoPickLocked(PickState* pick) {
|
|
354
337
|
subchannel_list_->subchannel(next_ready_index);
|
355
338
|
GPR_ASSERT(sd->connected_subchannel() != nullptr);
|
356
339
|
pick->connected_subchannel = sd->connected_subchannel()->Ref();
|
357
|
-
if (pick->user_data != nullptr) {
|
358
|
-
*pick->user_data = sd->user_data();
|
359
|
-
}
|
360
340
|
if (grpc_lb_round_robin_trace.enabled()) {
|
361
341
|
gpr_log(GPR_INFO,
|
362
342
|
"[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
|
@@ -667,9 +647,9 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
|
|
667
647
|
|
668
648
|
void RoundRobin::UpdateLocked(const grpc_channel_args& args,
|
669
649
|
grpc_json* lb_config) {
|
670
|
-
const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES);
|
671
650
|
AutoChildRefsUpdater guard(this);
|
672
|
-
|
651
|
+
const ServerAddressList* addresses = FindServerAddressListChannelArg(&args);
|
652
|
+
if (addresses == nullptr) {
|
673
653
|
gpr_log(GPR_ERROR, "[RR %p] update provided no addresses; ignoring", this);
|
674
654
|
// If we don't have a current subchannel list, go into TRANSIENT_FAILURE.
|
675
655
|
// Otherwise, keep using the current subchannel list (ignore this update).
|
@@ -681,11 +661,9 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args,
|
|
681
661
|
}
|
682
662
|
return;
|
683
663
|
}
|
684
|
-
grpc_lb_addresses* addresses =
|
685
|
-
static_cast<grpc_lb_addresses*>(arg->value.pointer.p);
|
686
664
|
if (grpc_lb_round_robin_trace.enabled()) {
|
687
665
|
gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
|
688
|
-
this, addresses->
|
666
|
+
this, addresses->size());
|
689
667
|
}
|
690
668
|
// Replace latest_pending_subchannel_list_.
|
691
669
|
if (latest_pending_subchannel_list_ != nullptr) {
|
@@ -696,7 +674,7 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args,
|
|
696
674
|
}
|
697
675
|
}
|
698
676
|
latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
|
699
|
-
this, &grpc_lb_round_robin_trace, addresses, combiner(),
|
677
|
+
this, &grpc_lb_round_robin_trace, *addresses, combiner(),
|
700
678
|
client_channel_factory(), args);
|
701
679
|
// If we haven't started picking yet or the new list is empty,
|
702
680
|
// immediately promote the new list to the current list.
|
@@ -726,7 +704,7 @@ class RoundRobinFactory : public LoadBalancingPolicyFactory {
|
|
726
704
|
return OrphanablePtr<LoadBalancingPolicy>(New<RoundRobin>(args));
|
727
705
|
}
|
728
706
|
|
729
|
-
const char* name() const override { return
|
707
|
+
const char* name() const override { return kRoundRobin; }
|
730
708
|
};
|
731
709
|
|
732
710
|
} // namespace
|
@@ -26,6 +26,7 @@
|
|
26
26
|
#include <grpc/support/alloc.h>
|
27
27
|
|
28
28
|
#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
|
29
|
+
#include "src/core/ext/filters/client_channel/server_address.h"
|
29
30
|
#include "src/core/ext/filters/client_channel/subchannel.h"
|
30
31
|
#include "src/core/lib/channel/channel_args.h"
|
31
32
|
#include "src/core/lib/debug/trace.h"
|
@@ -141,8 +142,7 @@ class SubchannelData {
|
|
141
142
|
protected:
|
142
143
|
SubchannelData(
|
143
144
|
SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
|
144
|
-
const
|
145
|
-
const grpc_lb_address& address, grpc_subchannel* subchannel,
|
145
|
+
const ServerAddress& address, grpc_subchannel* subchannel,
|
146
146
|
grpc_combiner* combiner);
|
147
147
|
|
148
148
|
virtual ~SubchannelData();
|
@@ -156,9 +156,8 @@ class SubchannelData {
|
|
156
156
|
grpc_connectivity_state connectivity_state,
|
157
157
|
grpc_error* error) GRPC_ABSTRACT;
|
158
158
|
|
159
|
-
// Unrefs the subchannel.
|
160
|
-
|
161
|
-
virtual void UnrefSubchannelLocked(const char* reason);
|
159
|
+
// Unrefs the subchannel.
|
160
|
+
void UnrefSubchannelLocked(const char* reason);
|
162
161
|
|
163
162
|
private:
|
164
163
|
// Updates connected_subchannel_ based on pending_connectivity_state_unsafe_.
|
@@ -186,8 +185,7 @@ class SubchannelData {
|
|
186
185
|
|
187
186
|
// A list of subchannels.
|
188
187
|
template <typename SubchannelListType, typename SubchannelDataType>
|
189
|
-
class SubchannelList
|
190
|
-
: public InternallyRefCountedWithTracing<SubchannelListType> {
|
188
|
+
class SubchannelList : public InternallyRefCounted<SubchannelListType> {
|
191
189
|
public:
|
192
190
|
typedef InlinedVector<SubchannelDataType, 10> SubchannelVector;
|
193
191
|
|
@@ -226,15 +224,14 @@ class SubchannelList
|
|
226
224
|
// Note: Caller must ensure that this is invoked inside of the combiner.
|
227
225
|
void Orphan() override {
|
228
226
|
ShutdownLocked();
|
229
|
-
|
230
|
-
"shutdown");
|
227
|
+
InternallyRefCounted<SubchannelListType>::Unref(DEBUG_LOCATION, "shutdown");
|
231
228
|
}
|
232
229
|
|
233
230
|
GRPC_ABSTRACT_BASE_CLASS
|
234
231
|
|
235
232
|
protected:
|
236
233
|
SubchannelList(LoadBalancingPolicy* policy, TraceFlag* tracer,
|
237
|
-
const
|
234
|
+
const ServerAddressList& addresses, grpc_combiner* combiner,
|
238
235
|
grpc_client_channel_factory* client_channel_factory,
|
239
236
|
const grpc_channel_args& args);
|
240
237
|
|
@@ -279,8 +276,7 @@ class SubchannelList
|
|
279
276
|
template <typename SubchannelListType, typename SubchannelDataType>
|
280
277
|
SubchannelData<SubchannelListType, SubchannelDataType>::SubchannelData(
|
281
278
|
SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
|
282
|
-
const
|
283
|
-
const grpc_lb_address& address, grpc_subchannel* subchannel,
|
279
|
+
const ServerAddress& address, grpc_subchannel* subchannel,
|
284
280
|
grpc_combiner* combiner)
|
285
281
|
: subchannel_list_(subchannel_list),
|
286
282
|
subchannel_(subchannel),
|
@@ -490,19 +486,19 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
|
|
490
486
|
template <typename SubchannelListType, typename SubchannelDataType>
|
491
487
|
SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
|
492
488
|
LoadBalancingPolicy* policy, TraceFlag* tracer,
|
493
|
-
const
|
489
|
+
const ServerAddressList& addresses, grpc_combiner* combiner,
|
494
490
|
grpc_client_channel_factory* client_channel_factory,
|
495
491
|
const grpc_channel_args& args)
|
496
|
-
:
|
492
|
+
: InternallyRefCounted<SubchannelListType>(tracer),
|
497
493
|
policy_(policy),
|
498
494
|
tracer_(tracer),
|
499
495
|
combiner_(GRPC_COMBINER_REF(combiner, "subchannel_list")) {
|
500
496
|
if (tracer_->enabled()) {
|
501
497
|
gpr_log(GPR_INFO,
|
502
498
|
"[%s %p] Creating subchannel list %p for %" PRIuPTR " subchannels",
|
503
|
-
tracer_->name(), policy, this, addresses
|
499
|
+
tracer_->name(), policy, this, addresses.size());
|
504
500
|
}
|
505
|
-
subchannels_.reserve(addresses
|
501
|
+
subchannels_.reserve(addresses.size());
|
506
502
|
// We need to remove the LB addresses in order to be able to compare the
|
507
503
|
// subchannel keys of subchannels from a different batch of addresses.
|
508
504
|
// We also remove the inhibit-health-checking arg, since we are
|
@@ -510,19 +506,27 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
|
|
510
506
|
inhibit_health_checking_ = grpc_channel_arg_get_bool(
|
511
507
|
grpc_channel_args_find(&args, GRPC_ARG_INHIBIT_HEALTH_CHECKING), false);
|
512
508
|
static const char* keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
|
513
|
-
|
509
|
+
GRPC_ARG_SERVER_ADDRESS_LIST,
|
514
510
|
GRPC_ARG_INHIBIT_HEALTH_CHECKING};
|
515
511
|
// Create a subchannel for each address.
|
516
512
|
grpc_subchannel_args sc_args;
|
517
|
-
for (size_t i = 0; i < addresses
|
518
|
-
// If there were any balancer, we would have chosen grpclb
|
519
|
-
|
513
|
+
for (size_t i = 0; i < addresses.size(); i++) {
|
514
|
+
// If there were any balancer addresses, we would have chosen grpclb
|
515
|
+
// policy, which does not use a SubchannelList.
|
516
|
+
GPR_ASSERT(!addresses[i].IsBalancer());
|
520
517
|
memset(&sc_args, 0, sizeof(grpc_subchannel_args));
|
521
|
-
grpc_arg
|
522
|
-
|
518
|
+
InlinedVector<grpc_arg, 4> args_to_add;
|
519
|
+
args_to_add.emplace_back(
|
520
|
+
grpc_create_subchannel_address_arg(&addresses[i].address()));
|
521
|
+
if (addresses[i].args() != nullptr) {
|
522
|
+
for (size_t j = 0; j < addresses[i].args()->num_args; ++j) {
|
523
|
+
args_to_add.emplace_back(addresses[i].args()->args[j]);
|
524
|
+
}
|
525
|
+
}
|
523
526
|
grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove(
|
524
|
-
&args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove),
|
525
|
-
|
527
|
+
&args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove),
|
528
|
+
args_to_add.data(), args_to_add.size());
|
529
|
+
gpr_free(args_to_add[0].value.string);
|
526
530
|
sc_args.args = new_args;
|
527
531
|
grpc_subchannel* subchannel = grpc_client_channel_factory_create_subchannel(
|
528
532
|
client_channel_factory, &sc_args);
|
@@ -530,8 +534,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
|
|
530
534
|
if (subchannel == nullptr) {
|
531
535
|
// Subchannel could not be created.
|
532
536
|
if (tracer_->enabled()) {
|
533
|
-
char* address_uri =
|
534
|
-
grpc_sockaddr_to_uri(&addresses->addresses[i].address);
|
537
|
+
char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
|
535
538
|
gpr_log(GPR_INFO,
|
536
539
|
"[%s %p] could not create subchannel for address uri %s, "
|
537
540
|
"ignoring",
|
@@ -541,8 +544,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
|
|
541
544
|
continue;
|
542
545
|
}
|
543
546
|
if (tracer_->enabled()) {
|
544
|
-
char* address_uri =
|
545
|
-
grpc_sockaddr_to_uri(&addresses->addresses[i].address);
|
547
|
+
char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
|
546
548
|
gpr_log(GPR_INFO,
|
547
549
|
"[%s %p] subchannel list %p index %" PRIuPTR
|
548
550
|
": Created subchannel %p for address uri %s",
|
@@ -550,8 +552,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
|
|
550
552
|
address_uri);
|
551
553
|
gpr_free(address_uri);
|
552
554
|
}
|
553
|
-
subchannels_.emplace_back(this, addresses
|
554
|
-
addresses->addresses[i], subchannel, combiner);
|
555
|
+
subchannels_.emplace_back(this, addresses[i], subchannel, combiner);
|
555
556
|
}
|
556
557
|
}
|
557
558
|
|