ruby-kafka-ec2 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8620cd2b3622df81ce8105a45930522ff9c9b433efbcba2c356c52bd7bdc7df
-  data.tar.gz: 5ccbcce906e3f000d445fe3a2135f4da2d7f7dbbbe3c7267849fc59bd9aabc9d
+  metadata.gz: d731fe0da3282726a8bb898f98f0ab3fe0133463662d9e7bc3c536c965bfc278
+  data.tar.gz: e260eb39399056cc48cdf8b7b985b037cdb60eafc8f495771d7949e7eb581a52
 SHA512:
-  metadata.gz: d085d3c8a0bcd5a3e0ab00f1b89d3cba53af8f5a33e8bf78bbfa5e03e96c027a3a747ef34a730400e4567e3b6533615fc6477c57eb4164da2a3d9f1b43ab71cb
-  data.tar.gz: a1343b55e46e61346f1b5b179b15cfcc838ecfb39650063369fa9dd92c0b786b3872bcf0330bc7039ca66a856e9f3657e3249a6dc927460bb2f158186fb4930c
+  metadata.gz: 3f31a1d280d2f49b864fbc4068a4f198952180101973d2a93c708ff621238f83476dac72e8c302427f3413a6a9761f81b5a703b8020aee3edba5bf2a42e0a967
+  data.tar.gz: 9d4cda171d25754e42bda7a90c4d7b9c8a77fa8fc984a5546ccb1cbc10b4ccf10999f75c6dbaa7ecd4fdecdde285a3ccec0ead4071326564f6527382853edbe0

data/README.md

@@ -77,6 +77,48 @@ consumer = Kafka::EC2.with_assignment_strategy_factory(assignment_strategy_facto
 end
 ```
 
+You can also specify weights for each combination of availability zones and instance families:
+
+```ruby
+assignment_strategy_factory = Kafka::EC2::MixedInstanceAssignmentStrategyFactory.new(
+  weights: ->() {
+    db_cluster = rds.describe_db_clusters(filters: [
+      { name: "db-cluster-id", values: [ENV["RDS_CLUSTER"]] },
+    ]).db_clusters.first
+    db_instance_id = db_cluster.db_cluster_members.find { |m| m.is_cluster_writer }.db_instance_identifier
+    db_instance = rds.describe_db_instances(filters: [
+      { name: "db-cluster-id", values: [ENV["RDS_CLUSTER"]] },
+      { name: "db-instance-id", values: [db_instance_id] },
+    ]).db_instances.first
+
+    weights_for_writer_az = {
+      "r4" => 1.00,
+      "r5" => 1.20,
+      "m5" => 1.35,
+      "c5" => 1.50,
+    }
+    weights_for_other_az = {
+      "r4" => 0.40,
+      "r5" => 0.70,
+      "m5" => 0.80,
+      "c5" => 1.00,
+    }
+    if db_instance.availability_zone == "ap-northeast-1a"
+      {
+        "ap-northeast-1a" => weights_for_writer_az,
+        "ap-northeast-1c" => weights_for_other_az,
+      }
+    else
+      {
+        "ap-northeast-1a" => weights_for_other_az,
+        "ap-northeast-1c" => weights_for_writer_az,,
+      }
+    end
+  },
+)
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/kafka/ec2/mixed_instance_assignment_strategy.rb

@@ -18,10 +18,16 @@ module Kafka
       #   is the availability zone and whose value is the weight. If the object is a proc,
       #   it must returns such a hash and the proc is called every time the method "assign"
       #   is called.
-      def initialize(cluster:, instance_family_weights:, availability_zone_weights:)
+      # @param weights [Hash{String => Hash{String => Numeric}}, Proc] a hash whose the key
+      #   is the availability zone or the instance family and whose value is the hash like
+      #   instance_family_weights or availability_zone_weights. If the object is a proc,
+      #   it must returns such a hash and the proc is called every time the method "assign"
+      #   is called.
+      def initialize(cluster:, instance_family_weights: {}, availability_zone_weights: {}, weights: {})
         @cluster = cluster
         @instance_family_weights = instance_family_weights
         @availability_zone_weights = availability_zone_weights
+        @weights = weights
       end
 
       # Assign the topic partitions to the group members.
@@ -38,12 +44,13 @@ module Kafka
 
         instance_family_to_capacity = @instance_family_weights.is_a?(Proc) ? @instance_family_weights.call() : @instance_family_weights
         az_to_capacity = @availability_zone_weights.is_a?(Proc) ? @availability_zone_weights.call() : @availability_zone_weights
+        weights = @weights.is_a?(Proc) ? @weights.call() : @weights
         members.each do |member_id|
           group_assignment[member_id] = Protocol::MemberAssignment.new
 
           instance_id, instance_type, az = member_id_to_metadata[member_id].split(",")
           instance_id_to_member_ids[instance_id] << member_id
-          capacity = calculate_capacity(instance_type, az, instance_family_to_capacity, az_to_capacity)
+          capacity = calculate_capacity(instance_type, az, instance_family_to_capacity, az_to_capacity, weights)
           instance_id_to_capacity[instance_id] += capacity
           total_capacity += capacity
         end
@@ -86,9 +93,11 @@ module Kafka
 
       private
 
-      def calculate_capacity(instance_type, az, instance_family_to_capacity, az_to_capacity)
+      def calculate_capacity(instance_type, az, instance_family_to_capacity, az_to_capacity, weights)
         instance_family, _ = instance_type.split(".")
-        instance_family_to_capacity.fetch(instance_family, 1) * az_to_capacity.fetch(az, 1)
+
+        capacity = weights.dig(az, instance_family) || weights.dig(instance_family, az)
+        capacity || instance_family_to_capacity.fetch(instance_family, 1) * az_to_capacity.fetch(az, 1)
       end
     end
   end

data/lib/kafka/ec2/mixed_instance_assignment_strategy_factory.rb

@@ -7,7 +7,9 @@ module Kafka
     class MixedInstanceAssignmentStrategyFactory
       # @param instance_family_weights [Hash, Proc]
       # @param availability_zone_weights [Hash, Proc]
-      def initialize(instance_family_weights: {}, availability_zone_weights: {})
+      # @param weights [Hash, Proc]
+      # @see Kafka::EC2::MixedInstanceAssignmentStrategy#initialize
+      def initialize(instance_family_weights: {}, availability_zone_weights: {}, weights: {})
         @instance_family_weights = instance_family_weights
         @availability_zone_weights = availability_zone_weights
       end

data/lib/kafka/ec2/version.rb

@@ -1,5 +1,5 @@
 module Kafka
   class EC2
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka-ec2
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - abicky
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-kafka