ruby-kafka-ec2 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/kafka/ec2/mixed_instance_assignment_strategy.rb +14 -2
- data/lib/kafka/ec2/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 66c71213189c16f43593597889adfb2ef3d0f4757cbf6ae7bb600310a2f88855
|
|
4
|
+
data.tar.gz: 71a0256485b92b88ed891e76bd3c3cbd85420b70332835cd33d4aeaf748e207d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c88ff1e2fe4ebd92fe6b9a13a87fd5e9582c09228ae17b29cdbd4c0186f83e22d6c96bb6f193fbd5ae2ed624979fe3c33a80c6db88e72a0800b2f878ffbfc7b1
|
|
7
|
+
data.tar.gz: 78cd8b945be174b64cdd261686683521160ed7c2c7334c6586726d1803abea4e0aad96d8470304ef324fe7b04281b7048d1591794771178c2bf71d0597f07ca7
|
|
@@ -42,6 +42,7 @@ module Kafka
|
|
|
42
42
|
instance_id_to_capacity = Hash.new(0)
|
|
43
43
|
instance_id_to_member_ids = Hash.new { |h, k| h[k] = [] }
|
|
44
44
|
total_capacity = 0
|
|
45
|
+
member_id_to_instance_id = {}
|
|
45
46
|
|
|
46
47
|
instance_family_to_capacity = @instance_family_weights.is_a?(Proc) ? @instance_family_weights.call() : @instance_family_weights
|
|
47
48
|
az_to_capacity = @availability_zone_weights.is_a?(Proc) ? @availability_zone_weights.call() : @availability_zone_weights
|
|
@@ -51,6 +52,7 @@ module Kafka
|
|
|
51
52
|
|
|
52
53
|
instance_id, instance_type, az = member_id_to_metadata[member_id].split(",")
|
|
53
54
|
instance_id_to_member_ids[instance_id] << member_id
|
|
55
|
+
member_id_to_instance_id[member_id] = instance_id
|
|
54
56
|
capacity = calculate_capacity(instance_type, az, instance_family_to_capacity, az_to_capacity, weights)
|
|
55
57
|
instance_id_to_capacity[instance_id] += capacity
|
|
56
58
|
total_capacity += capacity
|
|
@@ -70,6 +72,7 @@ module Kafka
|
|
|
70
72
|
|
|
71
73
|
last_index = 0
|
|
72
74
|
member_id_to_acceptable_partition_weight = {}
|
|
75
|
+
instance_id_to_total_acceptable_partition_weight = Hash.new(0)
|
|
73
76
|
instance_id_to_capacity.each do |instance_id, capacity|
|
|
74
77
|
member_ids = instance_id_to_member_ids[instance_id]
|
|
75
78
|
member_ids.each do |member_id|
|
|
@@ -86,15 +89,24 @@ module Kafka
|
|
|
86
89
|
end
|
|
87
90
|
|
|
88
91
|
member_id_to_acceptable_partition_weight[member_id] = acceptable_partition_weight
|
|
92
|
+
instance_id_to_total_acceptable_partition_weight[instance_id] += acceptable_partition_weight
|
|
89
93
|
end
|
|
90
94
|
end
|
|
91
95
|
|
|
92
96
|
while last_index < topic_partitions.size
|
|
93
|
-
|
|
97
|
+
max_acceptable_partition_weight = member_id_to_acceptable_partition_weight.values.max
|
|
98
|
+
member_ids = member_id_to_acceptable_partition_weight.select { |_, w| w == max_acceptable_partition_weight }.keys
|
|
99
|
+
if member_ids.size == 1
|
|
100
|
+
member_id = member_ids.first
|
|
101
|
+
else
|
|
102
|
+
member_id = member_ids.max_by { |id| instance_id_to_total_acceptable_partition_weight[member_id_to_instance_id[id]] }
|
|
103
|
+
end
|
|
94
104
|
topic, partition = topic_partitions[last_index]
|
|
95
105
|
group_assignment[member_id].assign(topic, [partition])
|
|
96
106
|
|
|
97
|
-
|
|
107
|
+
partition_weight = partition_weights.dig(topic, partition)
|
|
108
|
+
member_id_to_acceptable_partition_weight[member_id] -= partition_weight
|
|
109
|
+
instance_id_to_total_acceptable_partition_weight[member_id_to_instance_id[member_id]] -= partition_weight
|
|
98
110
|
|
|
99
111
|
last_index += 1
|
|
100
112
|
end
|
data/lib/kafka/ec2/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ruby-kafka-ec2
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- abicky
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-03-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: ruby-kafka
|
|
@@ -116,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
116
116
|
- !ruby/object:Gem::Version
|
|
117
117
|
version: '0'
|
|
118
118
|
requirements: []
|
|
119
|
-
rubygems_version: 3.
|
|
119
|
+
rubygems_version: 3.1.4
|
|
120
120
|
signing_key:
|
|
121
121
|
specification_version: 4
|
|
122
122
|
summary: An extension of ruby-kafka for EC2
|