ktl 1.0.0-java → 1.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 57f0bf86bb51e0401585eaec3993a4fb971d00c6
-  data.tar.gz: 7270677f6a1e5d7550dc35cdd51b351ac293e4bd
+  metadata.gz: 350c93c4fed5af08036c7e9e867df6f7c54cf26a
+  data.tar.gz: b3b963caa62735ca426a2d46919fbd09a350d67b
 SHA512:
-  metadata.gz: 75986093b1f59323ff894d9400b2921a7e66a3da5a60acc4b2639a24118011fa52a7e6143cde031b751c6a68f9cb865c0bd0a43ac80589ab960b75706b8f02d6
-  data.tar.gz: 0ed21803a5f5b99cd7809a63307625cb27f85a970adef1e39be894da140d41c675e85e8243a14e57ac108a90cfc10cba5c25cb80b10382548bdf899759b3b803
+  metadata.gz: b4b1da24ea11e84cca3fbbf678fecd01cbfb98d955ced80d44927935df43702a29ebc477e880dc9ffa0369c46f3235e6dd3ffa2e7cb51a903a8a89124b8b9ae6
+  data.tar.gz: 7d24168d5cde8623362a4e9873eb1b33e118c2e15c07975e7c6abbebc23a52911ef8878152cae2052d30b2c8ba36c2603afec39b52c32f0997a620f4f8511998

data/lib/ext/kafka.rb

@@ -12,6 +12,9 @@ end
 
 module ZkClient
   java_import 'org.I0Itec.zkclient.ZkClient'
+  java_import 'org.I0Itec.zkclient.IZkStateListener'
+  java_import 'org.I0Itec.zkclient.IZkDataListener'
+  java_import 'org.I0Itec.zkclient.IZkChildListener'
 
   module Exception
     include_package 'org.I0Itec.zkclient.exception'
@@ -127,6 +130,23 @@ module Kafka
       )
       Scala::Collection::JavaConversions.seq_as_java_list(broker_metadatas).to_a
     end
+
+    def self.get_broker_rack(zk_client, broker_id)
+      broker_metadata = Kafka::Admin.get_broker_metadatas(zk_client, [broker_id]).first
+      if broker_metadata
+        rack = broker_metadata.rack
+        unless rack.defined?
+          raise "Broker #{broker_metadata.id} is missing rack information, unable to create rack aware shuffle plan."
+        end
+        rack.get
+      end
+    rescue Java::KafkaAdmin::AdminOperationException => e
+      if e.message.include? '--disable-rack-aware'
+        raise "Not all brokers have rack information. Unable to create rack aware shuffle plan."
+      else
+        raise e
+      end
+    end
   end
 
   module Protocol

data/lib/ktl.rb

@@ -42,6 +42,7 @@ require 'ktl/cluster_stats_task'
 require 'ktl/decommission_plan'
 require 'ktl/migration_plan'
 require 'ktl/reassigner'
+require 'ktl/continous_reassigner'
 require 'ktl/reassignment_progress'
 require 'ktl/reassignment_task'
 require 'ktl/shuffle_plan'

data/lib/ktl/cluster.rb

@@ -28,16 +28,21 @@ module Ktl
     end
 
     desc 'migrate-broker', 'Migrate partitions from one broker to another'
-    option :from, aliases: %w[-f], type: :numeric, required: true, desc: 'Broker ID of old leader'
-    option :to, aliases: %w[-t], type: :numeric, required: true, desc: 'Broker ID of new leader'
+    option :from, aliases: %w[-f], type: :array, required: true, desc: 'Broker IDs to migrate away from'
+    option :to, aliases: %w[-t], type: :array, required: true, desc: 'Broker IDs to migrate to'
     option :zookeeper, aliases: %w[-z], required: true, desc: 'ZooKeeper URI'
     option :limit, aliases: %w[-l], type: :numeric, desc: 'Max number of partitions to reassign at a time'
+    option :verbose, aliases: %w[-v], desc: 'Verbose output'
+    option :dryrun, aliases: %w[-d], desc: 'Output reassignment plan without executing'
+    option :wait, aliases: %w[-w], type: :boolean, desc: 'Wait for all reassignments to finish'
+    option :delay, type: :numeric, desc: 'Delay in seconds between continous reassignment iterations, default 5s'
+    option :multi_step_migration, type: :boolean, default: true, desc: 'Perform migration in multiple steps, mirroring partitions to new brokers before removing the old'
     def migrate_broker
       with_zk_client do |zk_client|
-        old_leader, new_leader = options.values_at(:from, :to)
-        plan = MigrationPlan.new(zk_client, old_leader, new_leader)
-        reassigner = Reassigner.new(zk_client, limit: options.limit, logger: logger)
-        execute_reassignment(reassigner, plan)
+        old_brokers, new_brokers = options.values_at(:from, :to)
+        plan = MigrationPlan.new(zk_client, old_brokers.map(&:to_i), new_brokers.map(&:to_i), log_plan: options.verbose, logger: logger)
+        reassigner = create_reassigner(zk_client, options)
+        execute_reassignment(reassigner, plan, options)
       end
     end
 
@@ -51,6 +56,9 @@ module Ktl
     option :zookeeper, aliases: %w[-z], required: true, desc: 'ZooKeeper URI'
     option :verbose, aliases: %w[-v], desc: 'Verbose output'
     option :dryrun, aliases: %w[-d], desc: 'Output reassignment plan without executing'
+    option :wait, aliases: %w[-w], type: :boolean, desc: 'Wait for all reassignments to finish'
+    option :delay, type: :numeric, desc: 'Delay in seconds between continous reassignment iterations, default 5s'
+    option :multi_step_migration, type: :boolean, default: true, desc: 'Perform migration in multiple steps, mirroring partitions to new brokers before removing the old'
     def shuffle(regexp='.*')
       with_zk_client do |zk_client|
         plan_factory = if options.rack_aware
@@ -68,8 +76,8 @@ module Ktl
           logger: logger,
           log_plan: options.dryrun,
         })
-        reassigner = Reassigner.new(zk_client, limit: options.limit, logger: logger, log_assignments: options.verbose)
-        execute_reassignment(reassigner, plan, options.dryrun)
+        reassigner = create_reassigner(zk_client, options)
+        execute_reassignment(reassigner, plan, options)
       end
     end
 
@@ -77,6 +85,10 @@ module Ktl
     option :limit, aliases: %w[-l], type: :numeric, desc: 'Max number of partitions to reassign at a time'
     option :rendezvous, aliases: %w[-R], type: :boolean, desc: 'Whether to use Rendezvous-hashing'
     option :zookeeper, aliases: %w[-z], required: true, desc: 'ZooKeeper URI'
+    option :verbose, aliases: %w[-v], desc: 'Verbose output'
+    option :dryrun, aliases: %w[-d], desc: 'Output reassignment plan without executing'
+    option :wait, aliases: %w[-w], type: :boolean, desc: 'Wait for all reassignments to finish'
+    option :delay, type: :numeric, desc: 'Delay in seconds between continous reassignment iterations, default 5s'
     def decommission_broker(broker_id)
       with_zk_client do |zk_client|
         if options.rendezvous?
@@ -84,8 +96,8 @@ module Ktl
         else
           plan = DecommissionPlan.new(zk_client, broker_id.to_i)
         end
-        reassigner = Reassigner.new(zk_client, limit: options.limit, logger: logger)
-        execute_reassignment(reassigner, plan)
+        reassigner = create_reassigner(zk_client, options)
+        execute_reassignment(reassigner, plan, options)
       end
     end
 
@@ -101,8 +113,16 @@ module Ktl
 
     private
 
-    def execute_reassignment(reassigner, plan, dryrun = false)
-      ReassignmentTask.new(reassigner, plan, shell, logger: logger).execute(dryrun)
+    def execute_reassignment(reassigner, plan, options)
+      ReassignmentTask.new(reassigner, plan, shell, logger: logger).execute(options.dryrun)
+    end
+
+    def create_reassigner(zk_client, options)
+      if options.wait?
+        ContinousReassigner.new(zk_client, limit: options.limit, logger: logger, log_assignments: options.verbose, delay: options.delay, shell: shell, multi_step_migration: options.multi_step_migration)
+      else
+        Reassigner.new(zk_client, limit: options.limit, logger: logger, log_assignments: options.verbose, multi_step_migration: options.multi_step_migration)
+      end
     end
   end
 end

data/lib/ktl/cluster_stats_task.rb

@@ -19,7 +19,7 @@ module Ktl
       brokers.foreach do |broker|
         leader_for = ownership[broker.id]
         share = leader_for.fdiv(partitions.size.to_f) * 100
-        @shell.say '    - %d leader for %d partitions (%.2f %%)' % [broker.id, leader_for, share]
+        @shell.say '    - %s leader for %d partitions (%.2f %%)' % [broker.to_s, leader_for, share]
       end
     end
 

data/lib/ktl/continous_reassigner.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+module Ktl
+  class ContinousReassigner < Reassigner
+    include ZkClient::IZkDataListener
+
+    def initialize(zk_client, options={})
+      super(zk_client, options)
+      @latch = JavaConcurrent::CountDownLatch.new(1)
+      @sleeper = options[:sleeper] || java.lang.Thread
+      @delay = options[:delay] || 5
+      @shell = options[:shell]
+    end
+
+    def execute(reassignment)
+      Signal.trap('SIGINT', proc { @logger.info 'Exiting due to Ctrl-C'; @latch.count_down })
+      @zk_client.watch_data(zk_utils.class.reassign_partitions_path, self)
+      if reassignment_in_progress?
+        @logger.info 'reassignment already in progress, watching for changes...'
+        progress = ReassignmentProgress.new(@zk_client, logger: @logger, verbose: true)
+        progress.display(@shell)
+      else
+        reassign(reassignment)
+      end
+      @latch.await
+    end
+
+    def handle_data_change(path, data)
+      parsed_data = JSON.parse(data)
+      if (partitions = parsed_data['partitions'])
+        partitions = partitions.map { |r| r.values_at('topic', 'partition').join(':') }
+        @logger.debug sprintf('%d partitions left to reassign (%p)', partitions.size, partitions.size <= 5 ? partitions : '...')
+      else
+        @logger.info sprintf('Data without `partitions` key: %p', parsed_data)
+      end
+    rescue => e
+      @logger.error sprintf('Bad data: %p', data)
+    end
+
+    def handle_data_deleted(path)
+      reassignment = load_overflow
+      if reassignment.empty?
+        @zk_client.unsubscribe_data(zk_utils.class.reassign_partitions_path, self)
+        delete_previous_state
+        @latch.count_down
+      else
+        @logger.info sprintf('Waiting %ds before next assignment', @delay)
+        @sleeper.sleep(@delay * 1000)
+        reassign(reassignment)
+      end
+    end
+  end
+end

data/lib/ktl/migration_plan.rb

@@ -2,10 +2,22 @@
 
 module Ktl
   class MigrationPlan
-    def initialize(zk_client, old_leader, new_leader)
+    def initialize(zk_client, from_brokers, to_brokers, options = {})
       @zk_client = zk_client
-      @old_leader = old_leader.to_java
-      @new_leader = new_leader.to_java
+      @from_brokers = from_brokers
+      @to_brokers = to_brokers
+      if @from_brokers.length != @to_brokers.length
+        raise ArgumentError, "Both brokers lists must be of equal length. From: #{@from_brokers}, To: #{@to_brokers}"
+      elsif !(@from_brokers & @to_brokers).empty?
+        raise ArgumentError, "Broker lists must be mutually exclusive. From: #{@from_brokers}, To: #{@to_brokers}"
+      end
+      from_racks = from_brokers.map {|broker_id| Kafka::Admin.get_broker_rack(zk_client, broker_id)}
+      to_racks = to_brokers.map {|broker_id| Kafka::Admin.get_broker_rack(zk_client, broker_id)}
+      if from_racks != to_racks && from_racks.compact.any?
+        raise ArgumentError, "Both broker lists must have the same rack setup. From: #{from_racks}, To: #{to_racks}"
+      end
+      @logger = options[:logger] || NullLogger.new
+      @log_plan = !!options[:log_plan]
     end
 
     def generate
@@ -15,9 +27,16 @@ module Ktl
       assignments.each do |item|
         topic_partition = item.first
         replicas = item.last
-        if replicas.contains?(@old_leader)
-          index = replicas.index_of(@old_leader)
-          new_replicas = replicas.updated(index, @new_leader, CanBuildFrom)
+        new_replicas = replicas
+        @from_brokers.each_with_index do |from_broker, index|
+          to_broker = @to_brokers[index]
+          if new_replicas.contains?(from_broker)
+            replacement_index = new_replicas.index_of(from_broker)
+            new_replicas = new_replicas.updated(replacement_index, to_broker, CanBuildFrom)
+          end
+        end
+        if replicas != new_replicas
+          @logger.debug "Moving #{topic_partition.topic},#{topic_partition.partition} from #{replicas} to #{new_replicas}" if @log_plan
           plan += Scala::Tuple.new(topic_partition, new_replicas)
         end
       end

data/lib/ktl/reassigner.rb

@@ -2,6 +2,9 @@
 
 module Ktl
   class Reassigner
+
+    attr_reader :limit
+
     def initialize(zk_client, options={})
       @zk_client = zk_client
       @limit = options[:limit]
@@ -9,6 +12,7 @@ module Ktl
       @state_path = '/ktl/reassign'
       @logger = options[:logger] || NullLogger.new
       @log_assignments = !!options[:log_assignments]
+      @multi_step_migration = options[:multi_step_migration]
     end
 
     def reassignment_in_progress?
@@ -31,29 +35,72 @@ module Ktl
         data = parse_reassignment_json(overflow_json)
         overflow = overflow.send('++', data)
       end
-      delete_previous_overflow
       overflow
+    rescue ZkClient::Exception::ZkNoNodeException
+      Scala::Collection::Map.empty
     end
 
     def execute(reassignment)
+      reassign(reassignment)
+    end
+
+    private
+
+    JSON_MAX_SIZE = 1024**2
+
+    def reassign(reassignment)
+      if (limit)
+        @logger.info 'reassigning %d of %d partitions' % [limit, reassignment.size]
+      else
+        @logger.info 'reassigning %d partitions' % reassignment.size
+      end
+
       reassignments = split(reassignment, @limit)
-      actual_reassignment = reassignments.shift
-      if @log_assignments
-        Scala::Collection::JavaConversions.as_java_iterable(actual_reassignment).each do |pr|
-          topic_and_partition, replicas = pr.elements
+      reassignment_candidates = reassignments.shift
+      actual_reassignment = Scala::Collection::Map.empty
+      next_step_assignments = Scala::Collection::Map.empty
+      Scala::Collection::JavaConversions.as_java_iterable(reassignment_candidates).each do |pr|
+        topic_and_partition, replicas = pr.elements
+        if @multi_step_migration && step1_replicas = is_two_step_operation(topic_and_partition, replicas)
+          if step1_replicas.uniq != step1_replicas
+            raise "Multiple replicas on the same broker, this should not happen... #{step1_replicas}"
+          end
+          step1_replicas = Scala::Collection::JavaConversions.as_scala_iterable(step1_replicas)
+          next_step_assignments += pr
+          actual_reassignment += Scala::Tuple.new(topic_and_partition, step1_replicas)
+          brokers = Scala::Collection::JavaConversions.as_java_iterable(step1_replicas).to_a
+          eventual_brokers = Scala::Collection::JavaConversions.as_java_iterable(replicas).to_a
+          @logger.debug "Mirroring #{topic_and_partition.topic},#{topic_and_partition.partition} to #{brokers.join(',')} for eventual transition to #{eventual_brokers.join(',')}" if @log_assignments
+        else
+          actual_reassignment += pr
           brokers = Scala::Collection::JavaConversions.as_java_iterable(replicas).to_a
-          @logger.info "Assigning #{topic_and_partition.topic},#{topic_and_partition.partition} to #{brokers.join(',')}"
+          @logger.debug "Assigning #{topic_and_partition.topic},#{topic_and_partition.partition} to #{brokers.join(',')}" if @log_assignments
         end
       end
       json = reassignment_json(actual_reassignment)
       @zk_client.reassign_partitions(json)
-      manage_overflow(reassignments)
+      manage_overflow(split(next_step_assignments, nil) + reassignments)
       manage_progress_state(actual_reassignment)
     end
 
-    private
+    def is_two_step_operation(topic_and_partition, final_replicas)
+      replicas = Scala::Collection::JavaConversions.as_java_iterable(final_replicas).to_a.uniq
+      topic_list = Scala::Collection::JavaConversions.as_scala_iterable([topic_and_partition.topic])
+      assignments = ScalaEnumerable.new(@zk_client.replica_assignment_for_topics(topic_list))
+      assignments.each do |item|
+        item_topic_partition = item.first
+        if item_topic_partition.partition == topic_and_partition.partition
+          item_replicas = Scala::Collection::JavaConversions.as_java_iterable(item.last).to_a.uniq
+          diff_replicas = replicas - item_replicas
+          unless diff_replicas.empty?
+            transition_replicas = item_replicas + diff_replicas
+            return transition_replicas
+          end
+        end
+      end
 
-    JSON_MAX_SIZE = 1024**2
+      false
+    end
 
     def manage_progress_state(reassignment)
       delete_previous_state
@@ -73,6 +120,9 @@ module Ktl
         @zk_client.delete_znode(overflow_path(index))
       end
     rescue ZkClient::Exception::ZkNoNodeException
+      # no-op
+    rescue => e
+      puts e.backtrace.join($/)
     end
 
     def manage_overflow(reassignments)

data/lib/ktl/reassignment_task.rb

@@ -11,7 +11,12 @@ module Ktl
 
     def execute(dryrun = false)
       if @reassigner.reassignment_in_progress?
-        @logger.warn 'reassignment already in progress, exiting'
+        if @reassigner.is_a?(ContinousReassigner)
+          reassignment = @reassigner.load_overflow
+          @reassigner.execute(reassignment)
+        else
+          @logger.warn 'reassignment already in progress, exiting'
+        end
       else
         if use_overflow?
           @logger.info 'loading overflow data'
@@ -21,7 +26,6 @@ module Ktl
           reassignment = @plan.generate
         end
         if reassignment.size > 0
-          @logger.info 'reassigning %d partitions' % reassignment.size
           if dryrun
             @logger.info 'dryrun detected, skipping reassignment'
           else
@@ -43,4 +47,3 @@ module Ktl
     end
   end
 end
-

data/lib/ktl/shuffle_plan.rb

@@ -55,9 +55,11 @@ module Ktl
     end
 
     def assign_replicas_to_brokers(topic, brokers, partition_count, replica_count)
-      broker_metadatas = brokers.map { |x| Kafka::Admin::BrokerMetadata.new(x.to_java(:int), Scala::Option[nil]) }
-      broker_metadatas = Scala::Collection::JavaConversions.as_scala_iterable(broker_metadatas).to_seq
-      Kafka::Admin.assign_replicas_to_brokers(broker_metadatas, partition_count, replica_count)
+      @broker_metadatas ||= begin
+        broker_metadatas = Kafka::Admin.get_broker_metadatas(@zk_client, brokers)
+        Scala::Collection::JavaConversions.as_scala_iterable(broker_metadatas).to_seq
+      end
+      Kafka::Admin.assign_replicas_to_brokers(@broker_metadatas, partition_count, replica_count)
     rescue Kafka::Admin::AdminOperationException => e
       raise ArgumentError, sprintf('%s (%s)', e.message, e.class.name), e.backtrace
     end
@@ -125,21 +127,7 @@ module Ktl
     private
 
     def rack_for(broker_id)
-      unless @rack_mappings[broker_id]
-        broker_metadata = Kafka::Admin.get_broker_metadatas(@zk_client, [broker_id]).first
-        rack = broker_metadata.rack
-        unless rack.isDefined
-          raise "Broker #{broker_metadata.id} is missing rack information, unable to create rack aware shuffle plan."
-        end
-        @rack_mappings[broker_id] = rack.get
-      end
-      @rack_mappings[broker_id]
-    rescue Java::KafkaAdmin::AdminOperationException => e
-      if e.message.match '--disable-rack-aware'
-        raise "Not all brokers have rack information. Unable to create rack aware shuffle plan."
-      else
-        raise e
-      end
+      @rack_mappings[broker_id] ||= Kafka::Admin.get_broker_rack(@zk_client, broker_id)
     end
   end
 end

data/lib/ktl/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Ktl
-  VERSION = '1.0.0'.freeze
+  VERSION = '1.1.0'.freeze
 end

data/lib/ktl/zookeeper_client.rb

@@ -84,6 +84,22 @@ module Ktl
       @utils.path_exists(path)
     end
 
+    def watch_state(path, listener)
+      zk_client.subscribe_state_changes(path, listener)
+    end
+
+    def watch_data(path, listener)
+      zk_client.subscribe_data_changes(path, listener)
+    end
+
+    def watch_child(path, listener)
+      zk_client.subscribe_child_changes(path, listener)
+    end
+
+    def unsubscribe_data(path, listener)
+      zk_client.unsubscribe_data_changes(path, listener)
+    end
+
     private
 
     CONCURRENCY = 8
@@ -107,5 +123,9 @@ module Ktl
         acc.send('++', v)
       end
     end
+
+    def zk_client
+      @zk_client ||= @utils.class.create_zk_client(@uri, 5_000, 5_000)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ktl
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: java
 authors:
 - Burt Platform Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-15 00:00:00.000000000 Z
+date: 2017-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,6 +64,7 @@ files:
 - lib/ktl/cluster.rb
 - lib/ktl/cluster_stats_task.rb
 - lib/ktl/command.rb
+- lib/ktl/continous_reassigner.rb
 - lib/ktl/decommission_plan.rb
 - lib/ktl/migration_plan.rb
 - lib/ktl/reassigner.rb