gizzmo 0.11.2 → 0.11.3

data/VERSION

@@ -1 +1 @@
-0.11.2
+0.11.3

data/gizzmo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gizzmo}
-  s.version = "0.11.2"
+  s.version = "0.11.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2011-01-06}
+  s.date = %q{2011-01-26}
   s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.email = %q{kmaxwell@twitter.com}
   s.executables = ["gizzmo", "setup_shards"]
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
      "lib/gizzard/digest.rb",
      "lib/gizzard/migrator.rb",
      "lib/gizzard/nameserver.rb",
+     "lib/gizzard/rebalancer.rb",
      "lib/gizzard/shard_template.rb",
      "lib/gizzard/thrift.rb",
      "lib/gizzard/transformation.rb",
@@ -76,7 +77,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/twitter/gizzmo}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.test_files = [
     "test/gizzmo_spec.rb",
@@ -85,6 +86,9 @@ Gem::Specification.new do |s|
      "test/scheduler_spec.rb",
      "test/shard_template_spec.rb",
      "test/spec_helper.rb",
+     "test/test_server/target/gen-rb/test_server.rb",
+     "test/test_server/target/gen-rb/test_server_constants.rb",
+     "test/test_server/target/gen-rb/test_server_types.rb",
      "test/transformation_spec.rb"
   ]
 
@@ -92,7 +96,7 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
     else
     end
   else

data/lib/gizzard.rb

@@ -7,3 +7,4 @@ require "gizzard/shard_template"
 require "gizzard/migrator"
 require "gizzard/transformation"
 require "gizzard/digest"
+require "gizzard/rebalancer"

data/lib/gizzard/commands.rb

@@ -317,28 +317,6 @@ module Gizzard
     end
   end
 
-  class RepairCommand < Command
-    def run
-      args = @argv.dup.map{|a| a.split(/\s+/)}.flatten
-      pairs = []
-      loop do
-        a = args.shift
-        b = args.shift
-        break unless a && b
-        pairs << [a, b]
-      end
-      pairs.each do |master, slave|
-        puts "#{master} #{slave}"
-        mprefixes = manager.shards_for_hostname(master).map{|s| s.id.table_prefix}
-        sprefixes = manager.shards_for_hostname(slave).map{|s| s.id.table_prefix}
-        delta = mprefixes - sprefixes
-        delta.each do |prefix|
-          puts "gizzmo copy #{master}/#{prefix} #{slave}/#{prefix}"
-        end
-      end
-    end
-  end
-
   class WrapCommand < Command
     def self.derive_wrapper_shard_id(shard_info, wrapping_class_name)
       suffix = "_" + wrapping_class_name.split(".").last.downcase.gsub("shard", "")
@@ -366,84 +344,6 @@ module Gizzard
     end
   end
 
-  class RebalanceCommand < Command
-
-    class NamedArray < Array
-      attr_reader :name
-      def initialize(name)
-        @name = name
-      end
-    end
-
-    def run
-      help! "No shards specified" if @argv.empty?
-      shards = []
-      command_options.write_only_shard ||= "com.twitter.gizzard.shards.WriteOnlyShard"
-      additional_hosts = (command_options.hosts || "").split(/[\s,]+/)
-      exclude_hosts = (command_options.exclude_hosts || "").split(/[\s,]+/)
-      ids = @argv.map{|arg| ShardId.new(*arg.split("/")) rescue nil }.compact
-      by_host = ids.inject({}) do |memo, id|
-        memo[id.hostname] ||= NamedArray.new(id.hostname)
-        memo[id.hostname] << id
-        memo
-      end
-
-      additional_hosts.each do |host|
-        by_host[host] ||= NamedArray.new(host)
-      end
-
-      exclude_hosts.each do |host|
-        by_host[host] ||= NamedArray.new(host)
-      end
-
-      sets = by_host.values
-      exclude_sets = exclude_hosts.map{|host| by_host[host]}
-      target_sets = sets - exclude_sets
-
-      exclude_sets.each do |set|
-        while set.length > 0
-          sorted = target_sets.sort_by{|s| s.length}
-          shortest = sorted.first
-          shortest.push set.pop
-        end
-      end
-
-      exclude_sets.each do |set|
-        while set.length > 0
-          sorted = target_sets.sort_by{|s| s.length}
-          shortest = sorted.first
-          shortest.push set.pop
-        end
-      end
-
-      begin
-        sorted = target_sets.sort_by{|s| s.length }
-        longest = sorted.last
-        shortest = sorted.first
-        shortest.push longest.pop
-      end while longest.length > shortest.length + 1
-
-      shard_info = nil
-      sets.each do |set|
-        host = set.name
-        set.each do |id|
-          if id.hostname != host
-            shard_info ||= manager.get_shard(id)
-            old = id.to_unix
-            id.hostname = host
-            shards << [old, id.to_unix]
-          end
-        end
-      end
-
-      new_shards = shards.map{|(old, new)| new }
-
-      puts "gizzmo create #{shard_info.class_name} -s '#{shard_info.source_type}' -d '#{shard_info.destination_type}' #{new_shards.join(" ")}"
-      puts "gizzmo wrap #{command_options.write_only_shard} #{new_shards.join(" ")}"
-      shards.map { |(old, new)| puts "gizzmo copy #{old} #{new}" }
-    end
-  end
-
   class PairCommand < Command
     def run
       ids = []
@@ -791,17 +691,29 @@ module Gizzard
     end
   end
 
+  class TablesCommand < Command
+    def run
+      puts manager.list_tables.join(" ")
+    end
+  end
+
   class TopologyCommand < Command
     def run
-      templates = manager.manifest(*global_options.tables).templates.inject({}) do |h, (t, fs)|
+      manifest  = manager.manifest(*global_options.tables)
+      templates = manifest.templates.inject({}) do |h, (t, fs)|
         h.update t.to_config => fs
       end
 
       if command_options.forwardings
         templates.
-          inject([]) { |h, (t, fs)| fs.each { |f| h << [f.base_id, t] }; h }.
+          inject([]) { |h, (t, fs)| fs.each { |f| h << [f.inspect, t] }; h }.
           sort.
-          each { |a| puts "%25d\t%s" % a }
+          each { |a| puts "%s\t%s" % a }
+      elsif command_options.root_shards
+        templates.
+          inject([]) { |a, (t, fs)| fs.each { |f| a << [f.shard_id.inspect, t] }; a }.
+          sort.
+          each { |a| puts "%s\t%s" % a }
       else
         templates.
           map { |(t, fs)| [fs.length, t] }.
@@ -830,6 +742,11 @@ module Gizzard
 
       scheduler_options[:quiet] = be_quiet
 
+      if transformation.noop?
+        puts "Nothing to do!"
+        exit
+      end
+
       unless be_quiet
         puts transformation.inspect
         puts ""
@@ -869,6 +786,70 @@ module Gizzard
         transformations[transformation] = trees
       end
 
+      transformations.reject! {|t,trees| t.noop? or trees.empty? }
+
+      if transformations.empty?
+        puts "Nothing to do!"
+        exit
+      end
+
+      base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
+
+      unless be_quiet
+        transformations.sort.each do |transformation, trees|
+          puts transformation.inspect
+          puts "Applied to #{trees.length} shards:"
+          trees.keys.sort.each {|f| puts "  #{f.inspect}" }
+        end
+        puts ""
+      end
+
+      unless global_options.force
+        print "Continue? (y/n) "; $stdout.flush
+        exit unless $stdin.gets.chomp == "y"
+        puts ""
+      end
+
+      Gizzard.schedule! manager,
+                        base_name,
+                        transformations,
+                        scheduler_options
+    end
+  end
+
+  class RebalanceCommand < Command
+    def run
+      help!("must have an even number of arguments") unless @argv.length % 2 == 0
+
+      scheduler_options = command_options.scheduler_options || {}
+      manifest          = manager.manifest(*global_options.tables)
+      copy_wrapper      = scheduler_options[:copy_wrapper]
+      be_quiet          = global_options.force && command_options.quiet
+      transformations   = {}
+
+      scheduler_options[:quiet] = be_quiet
+
+      dest_templates_and_weights = {}
+
+      @argv.each_slice(2) do |(weight_s, to_template_s)|
+        to     = ShardTemplate.parse(to_template_s)
+        weight = weight_s.to_i
+
+        dest_templates_and_weights[to] = weight
+      end
+
+      transformations = global_options.tables.inject({}) do |all, table|
+        trees      = manifest.trees.reject {|(f, s)| f.table_id != table }
+        rebalancer = Rebalancer.new(trees, dest_templates_and_weights, copy_wrapper)
+
+        all.update(rebalancer.transformations) {|t,a,b| a.merge b }
+      end
+
+      if transformations.empty?
+        puts "Nothing to do!"
+        exit
+      end
+
       base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
 
       unless be_quiet

data/lib/gizzard/nameserver.rb

@@ -1,6 +1,17 @@
 module Gizzard
   Shard = Struct.new(:info, :children, :weight)
 
+  module ParallelMap
+    def parallel_map(enumerable, &block)
+      enumerable.map do |elem|
+        Thread.new { Thread.current[:result] = block.call(elem) }
+      end.map do |thread|
+        thread.join
+        thread[:result]
+      end
+    end
+  end
+
   class Shard
     class << self
       def canonical_table_prefix(enum, table_id = nil, base_prefix = "shard")
@@ -75,6 +86,7 @@ module Gizzard
   end
 
   class Nameserver
+    include ParallelMap
 
     DEFAULT_PORT    = 7917
     DEFAULT_RETRIES = 20
@@ -96,8 +108,16 @@ module Gizzard
       ids.map {|id| with_retry { client.get_shard(id) } }
     end
 
+    def reload_updated_forwardings
+      parallel_map all_clients do |c|
+        with_retry { c.reload_updated_forwardings }
+      end
+    end
+
     def reload_config
-      all_clients.each {|c| with_retry { c.reload_config } }
+      parallel_map all_clients do |c|
+        with_retry { c.reload_config }
+      end
     end
 
     def copy_shard(from_shard_id, to_shard_id)
@@ -154,11 +174,12 @@ module Gizzard
     end
 
     class Manifest
-      attr_reader :forwardings, :links, :shard_infos, :trees, :templates
+      include ParallelMap
 
+      attr_reader :forwardings, :links, :shard_infos, :trees, :templates
 
       def initialize(nameserver, table_ids)
-        states = table_ids.map {|id| nameserver.dump_nameserver(id) }
+        states = nameserver.dump_nameserver(table_ids)
 
         @forwardings = states.map {|s| s.forwardings }.flatten
 

data/lib/gizzard/rebalancer.rb

@@ -0,0 +1,125 @@
+require 'set'
+
+module Gizzard
+  class Rebalancer
+    TemplateAndTree = Struct.new(:template, :forwarding, :tree)
+    Bucket          = Struct.new(:template, :approx_shards, :set)
+
+    class Bucket
+      def balance; set.length - approx_shards end
+      def add(e); set.add(e) end
+      def merge(es); set.merge(es) end
+      def delete(e); set.delete(e) end
+    end
+
+    # steps for rebalancing.
+    #
+    # 1. get a list of forwarding/template associations
+    # 2. get a list of destination templates and weights
+    # 3. order shards by weight. (ascending or descending)
+    # 4. put shards in destinations based on reducing number of copies required.
+    # 5.
+
+    def initialize(forwardings_to_trees, dest_templates_and_weights, wrapper)
+      @copy_dest_wrapper = wrapper
+      @shards = forwardings_to_trees.map do |forwarding, tree|
+        TemplateAndTree.new(tree.template, forwarding, tree)
+      end.flatten
+
+      @dest_templates      = dest_templates_and_weights.keys
+
+      total_shards = @shards.length
+      total_weight = dest_templates_and_weights.values.inject {|a,b| a + b }
+
+      @result = dest_templates_and_weights.map do |template, weight|
+        weight_fraction = weight / total_weight.to_f
+        approx_shards   = total_shards * weight_fraction
+
+        Bucket.new template, approx_shards, Set.new
+      end
+    end
+
+    def home!
+
+      # list of [template, shards] in descending length of shards
+      templates_to_shards =
+        @shards.inject({}) do |h, shard|
+          (h[shard.template] ||= []) << shard; h
+        end.sort_by {|(_,ss)| ss.length * -1 }
+
+      templates_to_shards.each do |(template, shards)|
+        descendants = memoized_concrete_descendants(template)
+
+        most_similar_buckets = []
+        last_cost = nil
+
+        @result.each do |bucket|
+          cost      = (memoized_concrete_descendants(bucket.template) - descendants).length
+          last_cost = cost if last_cost.nil?
+
+          if cost == last_cost
+            most_similar_buckets << bucket
+          elsif cost < last_cost
+            last_cost = cost
+            most_similar_buckets = [bucket]
+          end
+        end
+
+        dest_bucket = most_similar_buckets.sort_by {|b| b.balance }.first
+
+        dest_bucket.merge shards
+      end
+    end
+
+    def rebalance!
+      while bucket_disparity > 1
+        ordered = ordered_buckets
+        move_shard ordered.first.template, ordered.last.set.each {|e| break e }
+      end
+    end
+
+    def transformations
+      return @transformations if @transformations
+
+      home!
+      rebalance!
+
+      @transformations = {}
+      @result.each do |bucket|
+        bucket.set.each do |shard|
+          trans = Transformation.new(shard.template, bucket.template, @copy_dest_wrapper)
+          forwardings_to_trees = (@transformations[trans] ||= {})
+
+          forwardings_to_trees.update(shard.forwarding => shard.tree)
+        end
+      end
+
+      @transformations.reject! {|t, _| t.noop? }
+      @transformations
+    end
+
+    def ordered_buckets
+      @result.sort_by {|bucket| bucket.balance }
+    end
+
+    def memoized_concrete_descendants(t)
+      @memoized_concrete_descendants ||= {}
+      @memoized_concrete_descendants[t] ||= t.concrete_descendants
+    end
+
+    def bucket_disparity
+      ordered = ordered_buckets
+      ordered.last.balance - ordered.first.balance
+    end
+
+    def move_shard(template, shard)
+      @result.each do |bucket|
+        if bucket.template == template
+          bucket.add shard
+        else
+          bucket.delete shard
+        end
+      end
+    end
+  end
+end