gizzmo 0.11.0 → 0.11.1

data/Rakefile

@@ -1,3 +1,4 @@
+ROOT_DIR = File.expand_path(File.dirname(__FILE__))
 require 'rubygems'
 require 'rake'
 
@@ -16,28 +17,22 @@ rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
 end
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
 
 begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
+  require 'spec/rake/spectask'
+  Spec::Rake::SpecTask.new(:spec) do |t|
+    spec_opts = File.expand_path('test/spec.opts', ROOT_DIR)
+    if File.exist? spec_opts
+      t.spec_opts = ['--options', "\"#{spec_opts}\""]
+    end
+    t.spec_files = FileList['test/**/*_spec.rb']
   end
 rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
+  $stderr.puts "RSpec required to run tests."
 end
 
 task :test do
-  puts 
+  puts
   puts "=" * 79
   puts "You might want to read the README before running tests."
   puts "=" * 79
@@ -45,7 +40,7 @@ task :test do
   exec File.join(File.dirname(__FILE__), "test", "test.sh")
 end
 
-task :default => :test
+task :default => :spec
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|

data/VERSION

@@ -1 +1 @@
-0.11.0
+0.11.1

data/bin/setup_shards

@@ -0,0 +1,173 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'ostruct'
+require 'optparse'
+require 'gizzard'
+
+require 'gizzard/nameserver'
+require 'gizzard/migrator'
+require 'gizzard/transformation'
+require 'gizzard/shard_template'
+
+def usage(parser, reason)
+  puts
+  puts "ERROR: #{reason}"
+  puts
+  puts parser
+  puts
+  exit 1
+end
+
+options = {
+  :config_filename => ENV['HOME'] + "/.topology.yml",
+  :dry_run => false,
+  :max_copies => 20,
+  :table_id => nil
+}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.separator "Example: #{$0} -f topology.yml"
+
+  opts.on("-f", "--config=FILENAME", "load database topology config (default: #{options[:config_filename]})") do |filename|
+    options[:config_filename] = filename
+  end
+  opts.on("-i", "--id=TABLE_ID", "table id to use (default: #{options[:table_id]})") do |table_id|
+    options[:table_id] = table_id.to_i
+  end
+  opts.on("-s", "--shards=N", "create N shards") do |n|
+    options[:total_shards] = n.to_i
+  end
+  opts.on("-t", "--table=TABLE_NAME", "table name") do |table_name|
+    options[:table_name] = table_name
+  end
+  opts.on("-c", "--class=CLASS_NAME", "shard class to create") do |class_name|
+    options[:shard_class] = class_name
+  end
+  opts.on("-n", "--dry-run", "don't actually send gizzard commands") do
+    options[:dry_run] = true
+  end
+  opts.on("-x", "--max=COPIES", "max concurrent copies (default: #{options[:max_copies]})") do |n|
+    options[:max_copies] = n.to_i
+  end
+end
+
+parser.parse!(ARGV)
+
+config = begin
+  YAML.load_file(options[:config_filename])
+rescue => e
+  puts "Exception while reading config file: #{e}"
+  {}
+end
+
+total_shards = options[:total_shards] || config['total_shards']
+usage(parser, "Must define total_shards or -s") unless total_shards
+
+table_name = options[:table_name] || config['table_name']
+usage(parser, "Must define table_name or -t") unless table_name
+
+migrator_config = Gizzard::MigratorConfig.new
+migrator_config.prefix = table_name
+migrator_config.table_id = options[:table_id]
+migrator_config.source_type = config['source_type'] || ''
+migrator_config.destination_type = config['destination_type'] || ''
+migrator_config.forwarding_space = config['forwarding_space'] || 2 ** 64
+migrator_config.forwarding_space_min = config['forwarding_space_min'] || 2 ** 63 * -1
+
+querying_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => false)
+mutating_nameserver = Gizzard::Nameserver.new(config['app_hosts'] || 'localhost', :dry_run => options[:dry_run])
+
+usage(parser, "Config file must contain a 'partitions' definition") unless config['partitions']
+
+# Where the magic happens:
+
+new_templates = config['partitions'].map { |p| Gizzard::ShardTemplate.from_config(migrator_config, p) }
+
+EXISTING_CONFIG_CACHE = File.expand_path(".existing_shards.marshal")
+
+if File.exist? EXISTING_CONFIG_CACHE
+  puts "Loading cache from previous operation. If this is not desired, delete the cache file at #{EXISTING_CONFIG_CACHE}"
+  manifest = Marshal.load(File.read(EXISTING_CONFIG_CACHE))
+  raise "error loading cache!" unless manifest.is_a? Gizzard::Manifest
+else
+  puts "Querying nameserver..."
+  manifest = Gizzard::Manifest.new(querying_nameserver, migrator_config)
+  File.open(EXISTING_CONFIG_CACHE, 'w') { |f| f.write Marshal.dump(manifest) }
+end
+
+migrator_config.manifest = manifest
+existing_map = manifest.template_map
+
+
+puts "\nCalculating changes...\n"
+
+
+migrator = Gizzard::Migrator.new(existing_map, new_templates, total_shards, migrator_config)
+
+puts "\nUNCHANGED:"
+pp migrator.unchanged_templates
+
+puts "\nSIMILAR:"
+pp migrator.similar_templates
+
+puts "\nNEW:"
+pp migrator.new_templates
+
+puts "\nUNRECOGNIZED:"
+pp migrator.unrecognized_templates
+
+unless (transformations = migrator.transformations).empty?
+  puts "\nTRANSFORMATIONS:"
+  transformations.each { |t| puts ""; p t }
+
+  printf "Apply migration? [yN] "; $stdout.flush
+
+  if $stdin.gets.chomp == 'y'
+    pages = transformations.inject([]) { |pages, t| pages.concat t.paginate(options[:max_copies]) }
+
+    pages.each_with_index do |page, page_idx|
+      puts "\nTRANSFORMATION #{page_idx + 1} / #{pages.length}:"
+
+      puts page.inspect(true)
+
+      t_start = Time.now
+
+      if page.must_copy?
+        puts "    Preparing nameserver for copies."
+        page.prepare! mutating_nameserver, migrator_config
+        mutating_nameserver.reload_forwardings
+
+        puts "    Scheduling copies."
+        page.copy! mutating_nameserver, migrator_config
+
+        puts "    Waiting for copies to finish."
+        page.wait_for_copies mutating_nameserver, migrator_config
+
+        puts "    Finalizing nameserver changes."
+        page.cleanup! mutating_nameserver, migrator_config
+        mutating_nameserver.reload_forwardings
+      else
+        puts "   Applying nameserver changes."
+        page.apply! mutating_nameserver, migrator_config
+      end
+
+      puts "--- Time Elapsed: %0.3f ---" % (Time.now - t_start).to_f
+    end
+
+    mutating_nameserver.reload_forwardings
+
+    puts "Done!"
+
+  else
+    puts "Aborting migration."
+  end
+
+else
+  puts "No migration needed."
+end
+
+# remove the existing config state since we're done, and the system is
+# in a known good state.
+File.unlink EXISTING_CONFIG_CACHE

data/lib/gizzard.rb

@@ -2,4 +2,8 @@ $: << File.dirname(__FILE__)
 module Gizzard; end
 require "gizzard/thrift"
 require "gizzard/commands"
+require "gizzard/nameserver"
+require "gizzard/shard_template"
+require "gizzard/migrator"
+require "gizzard/transformation"
 require "gizzard/digest"

data/lib/gizzard/commands.rb

@@ -1,29 +1,51 @@
 require "pp"
+require "set"
 require "digest/md5"
 
 module Gizzard
   class Command
-    include Thrift
 
     attr_reader :buffer
 
-    def self.run(command_name, global_options, argv, subcommand_options, log, service=nil)
-      command_class = Gizzard.const_get("#{classify(command_name)}Command")
-      service = command_class.make_service(global_options, log) if service.nil?
-      command = command_class.new(service, global_options, argv, subcommand_options)
-      command.run
-      if command.buffer && command_name = global_options.render.shift
-        run(command_name, global_options, command.buffer, OpenStruct.new, log, service)
+    class << self
+      def run(command_name, global_options, argv, subcommand_options, log)
+        command_class = Gizzard.const_get("#{classify(command_name)}Command")
+
+        @manager      ||= make_manager(global_options, log)
+        @job_injector ||= make_job_injector(global_options, log)
+
+        command = command_class.new(@manager, @job_injector, global_options, argv, subcommand_options)
+        command.run
+
+        if command.buffer && command_name = global_options.render.shift
+          run(command_name, global_options, command.buffer, OpenStruct.new, log)
+        end
+      end
+
+      def classify(string)
+        string.split(/\W+/).map{|s| s.capitalize }.join("")
+      end
+
+      def make_manager(global_options, log)
+        hosts = global_options.hosts.map {|h| [h, global_options.port].join(":") }
+
+        Nameserver.new(hosts, :retries => global_options.retry,
+                              :log     => log,
+                              :framed  => global_options.framed,
+                              :dry_run => global_options.dry)
       end
-    end
 
-    def self.classify(string)
-      string.split(/\W+/).map { |s| s.capitalize }.join("")
+      def make_job_injector(global_options, log)
+        RetryProxy.new global_options.retry,
+          JobInjector.new(global_options.hosts.first, global_options.injector_port, log, true, global_options.dry)
+      end
     end
 
-    attr_reader :service, :global_options, :argv, :command_options
-    def initialize(service, global_options, argv, command_options)
-      @service         = service
+    attr_reader :manager, :job_injector, :global_options, :argv, :command_options
+
+    def initialize(manager, job_injector, global_options, argv, command_options)
+      @manager      = manager
+      @job_injector    = job_injector
       @global_options  = global_options
       @argv            = argv
       @command_options = command_options
@@ -62,49 +84,35 @@ module Gizzard
     end
   end
 
-  class ShardCommand < Command
-    def self.make_service(global_options, log)
-      RetryProxy.new global_options.retry.to_i,
-        Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port, log, global_options.framed, global_options.dry)
-    end
-  end
-
-  class JobCommand < Command
-    def self.make_service(global_options, log)
-      RetryProxy.new global_options.retry.to_i  ,
-        Gizzard::Thrift::JobManager.new(global_options.host, global_options.port + 2, log, global_options.framed, global_options.dry)
-    end
-  end
-
-  class AddforwardingCommand < ShardCommand
+  class AddforwardingCommand < Command
     def run
       help! if argv.length != 3
       table_id, base_id, shard_id_text = argv
       shard_id = ShardId.parse(shard_id_text)
-      service.set_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
+      manager.set_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
     end
   end
 
-  class DeleteforwardingCommand < ShardCommand
+  class DeleteforwardingCommand < Command
     def run
       help! if argv.length != 3
       table_id, base_id, shard_id_text = argv
       shard_id = ShardId.parse(shard_id_text)
-      service.remove_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
+      manager.remove_forwarding(Forwarding.new(table_id.to_i, base_id.to_i, shard_id))
     end
   end
 
-  class HostsCommand < ShardCommand
+  class HostsCommand < Command
     def run
-      service.list_hostnames.map do |host|
+      manager.list_hostnames.map do |host|
         puts host
       end
     end
   end
 
-  class ForwardingsCommand < ShardCommand
+  class ForwardingsCommand < Command
     def run
-      service.get_forwardings().sort_by do |f|
+      manager.get_forwardings.sort_by do |f|
         [ ((f.table_id.abs << 1) + (f.table_id < 0 ? 1 : 0)), f.base_id ]
       end.reject do |forwarding|
         @command_options.table_ids && !@command_options.table_ids.include?(forwarding.table_id)
@@ -114,7 +122,7 @@ module Gizzard
     end
   end
 
-  class SubtreeCommand < ShardCommand
+  class SubtreeCommand < Command
     def run
       @roots = []
       argv.each do |arg|
@@ -128,7 +136,7 @@ module Gizzard
     end
 
     def roots_of(id)
-      links = service.list_upward_links(id)
+      links = manager.list_upward_links(id)
       if links.empty?
         [id]
       else
@@ -137,7 +145,7 @@ module Gizzard
     end
 
     def down(id, depth = 0)
-      service.list_downward_links(id).map do |link|
+      manager.list_downward_links(id).map do |link|
         printable = "  " * depth + link.down_id.to_unix
         output printable
         down(link.down_id, depth + 1)
@@ -145,21 +153,10 @@ module Gizzard
     end
   end
 
-  class ReloadCommand < ShardCommand
+  class ReloadCommand < Command
     def run
       if global_options.force || ask
-        if @argv
-          # allow hosts to be given on the command line
-          @argv.each do |hostname|
-            output hostname
-            opts = global_options.dup
-            opts.host = hostname
-            s = self.class.make_service(opts, global_options.log || "./gizzmo.log")
-            s.reload_forwardings
-          end
-        else
-          service.reload_forwardings
-        end
+        manager.reload_config
       else
         STDERR.puts "aborted"
       end
@@ -171,17 +168,17 @@ module Gizzard
     end
   end
 
-  class DeleteCommand < ShardCommand
+  class DeleteCommand < Command
     def run
       argv.each do |arg|
         id  = ShardId.parse(arg)
-        service.delete_shard(id)
+        manager.delete_shard(id)
         output id.to_unix
       end
     end
   end
 
-  class AddlinkCommand < ShardCommand
+  class AddlinkCommand < Command
     def run
       up_id, down_id, weight = argv
       help! if argv.length != 3
@@ -189,29 +186,29 @@ module Gizzard
       up_id = ShardId.parse(up_id)
       down_id = ShardId.parse(down_id)
       link = LinkInfo.new(up_id, down_id, weight)
-      service.add_link(link.up_id, link.down_id, link.weight)
+      manager.add_link(link.up_id, link.down_id, link.weight)
       output link.to_unix
     end
   end
 
-  class UnlinkCommand < ShardCommand
+  class UnlinkCommand < Command
     def run
       up_id, down_id = argv
       up_id = ShardId.parse(up_id)
       down_id = ShardId.parse(down_id)
-      service.remove_link(up_id, down_id)
+      manager.remove_link(up_id, down_id)
     end
   end
 
-  class UnwrapCommand < ShardCommand
+  class UnwrapCommand < Command
     def run
       shard_ids = argv
       help! "No shards specified" if shard_ids.empty?
       shard_ids.each do |shard_id_string|
         shard_id = ShardId.parse(shard_id_string)
 
-        upward_links = service.list_upward_links(shard_id)
-        downward_links = service.list_downward_links(shard_id)
+        upward_links = manager.list_upward_links(shard_id)
+        downward_links = manager.list_downward_links(shard_id)
 
         if upward_links.length == 0 or downward_links.length == 0
           STDERR.puts "Shard #{shard_id_string} must not be a root or leaf"
@@ -220,19 +217,19 @@ module Gizzard
 
         upward_links.each do |uplink|
           downward_links.each do |downlink|
-            service.add_link(uplink.up_id, downlink.down_id, uplink.weight)
+            manager.add_link(uplink.up_id, downlink.down_id, uplink.weight)
             new_link = LinkInfo.new(uplink.up_id, downlink.down_id, uplink.weight)
-            service.remove_link(uplink.up_id, uplink.down_id)
-            service.remove_link(downlink.up_id, downlink.down_id)
+            manager.remove_link(uplink.up_id, uplink.down_id)
+            manager.remove_link(downlink.up_id, downlink.down_id)
             output new_link.to_unix
           end
         end
-        service.delete_shard shard_id
+        manager.delete_shard shard_id
       end
     end
   end
 
-  class CreateCommand < ShardCommand
+  class CreateCommand < Command
     def run
       help! if argv.length < 2
       class_name, *shard_ids = argv
@@ -241,26 +238,26 @@ module Gizzard
       destination_type = command_options.destination_type || ""
       shard_ids.each do |id|
         shard_id = ShardId.parse(id)
-        service.create_shard(ShardInfo.new(shard_id, class_name, source_type, destination_type, busy))
-        service.get_shard(shard_id)
+        manager.create_shard(ShardInfo.new(shard_id, class_name, source_type, destination_type, busy))
+        manager.get_shard(shard_id)
         output shard_id.to_unix
       end
     end
   end
 
-  class LinksCommand < ShardCommand
+  class LinksCommand < Command
     def run
       shard_ids = @argv
       shard_ids.each do |shard_id_text|
         shard_id = ShardId.parse(shard_id_text)
         next if !shard_id
         unless command_options.down
-          service.list_upward_links(shard_id).each do |link_info|
+          manager.list_upward_links(shard_id).each do |link_info|
             output command_options.ids ? link_info.up_id.to_unix : link_info.to_unix
           end
         end
         unless command_options.up
-          service.list_downward_links(shard_id).each do |link_info|
+          manager.list_downward_links(shard_id).each do |link_info|
             output command_options.ids ? link_info.down_id.to_unix : link_info.to_unix
           end
         end
@@ -268,41 +265,41 @@ module Gizzard
     end
   end
 
-  class InfoCommand < ShardCommand
+  class InfoCommand < Command
     def run
       shard_ids = @argv
       shard_ids.each do |shard_id|
-        shard_info = service.get_shard(ShardId.parse(shard_id))
+        shard_info = manager.get_shard(ShardId.parse(shard_id))
         output shard_info.to_unix
       end
     end
   end
 
-  class MarkbusyCommand < ShardCommand
+  class MarkbusyCommand < Command
     def run
       shard_ids = @argv
       shard_ids.each do |shard_id|
         id = ShardId.parse(shard_id)
-        service.mark_shard_busy(id, 1)
-        shard_info = service.get_shard(id)
+        manager.mark_shard_busy(id, 1)
+        shard_info = manager.get_shard(id)
         output shard_info.to_unix
       end
     end
   end
 
-  class MarkunbusyCommand < ShardCommand
+  class MarkunbusyCommand < Command
     def run
       shard_ids = @argv
       shard_ids.each do |shard_id|
         id = ShardId.parse(shard_id)
-        service.mark_shard_busy(id, 0)
-        shard_info = service.get_shard(id)
+        manager.mark_shard_busy(id, 0)
+        shard_info = manager.get_shard(id)
         output shard_info.to_unix
       end
     end
   end
 
-  class RepairCommand < ShardCommand
+  class RepairCommand < Command
     def run
       args = @argv.dup.map{|a| a.split(/\s+/)}.flatten
       pairs = []
@@ -314,8 +311,8 @@ module Gizzard
       end
       pairs.each do |master, slave|
         puts "#{master} #{slave}"
-        mprefixes = service.shards_for_hostname(master).map{|s| s.id.table_prefix}
-        sprefixes = service.shards_for_hostname(slave).map{|s| s.id.table_prefix}
+        mprefixes = manager.shards_for_hostname(master).map{|s| s.id.table_prefix}
+        sprefixes = manager.shards_for_hostname(slave).map{|s| s.id.table_prefix}
         delta = mprefixes - sprefixes
         delta.each do |prefix|
           puts "gizzmo copy #{master}/#{prefix} #{slave}/#{prefix}"
@@ -324,7 +321,7 @@ module Gizzard
     end
   end
 
-  class WrapCommand < ShardCommand
+  class WrapCommand < Command
     def self.derive_wrapper_shard_id(shard_info, wrapping_class_name)
       suffix = "_" + wrapping_class_name.split(".").last.downcase.gsub("shard", "")
       ShardId.new("localhost", shard_info.id.table_prefix + suffix)
@@ -335,15 +332,15 @@ module Gizzard
       help! "No shards specified" if shard_ids.empty?
       shard_ids.each do |shard_id_string|
         shard_id   = ShardId.parse(shard_id_string)
-        shard_info = service.get_shard(shard_id)
-        service.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
+        shard_info = manager.get_shard(shard_id)
+        manager.create_shard(ShardInfo.new(wrapper_id = self.class.derive_wrapper_shard_id(shard_info, class_name), class_name, "", "", 0))
 
-        existing_links = service.list_upward_links(shard_id)
+        existing_links = manager.list_upward_links(shard_id)
         unless existing_links.include?(LinkInfo.new(wrapper_id, shard_id, 1))
-          service.add_link(wrapper_id, shard_id, 1)
+          manager.add_link(wrapper_id, shard_id, 1)
           existing_links.each do |link_info|
-            service.add_link(link_info.up_id, wrapper_id, link_info.weight)
-            service.remove_link(link_info.up_id, link_info.down_id)
+            manager.add_link(link_info.up_id, wrapper_id, link_info.weight)
+            manager.remove_link(link_info.up_id, link_info.down_id)
           end
         end
         output wrapper_id.to_unix
@@ -351,7 +348,7 @@ module Gizzard
     end
   end
 
-  class RebalanceCommand < ShardCommand
+  class RebalanceCommand < Command
 
     class NamedArray < Array
       attr_reader :name
@@ -413,7 +410,7 @@ module Gizzard
         host = set.name
         set.each do |id|
           if id.hostname != host
-            shard_info ||= service.get_shard(id)
+            shard_info ||= manager.get_shard(id)
             old = id.to_unix
             id.hostname = host
             shards << [old, id.to_unix]
@@ -429,11 +426,11 @@ module Gizzard
     end
   end
 
-  class PairCommand < ShardCommand
+  class PairCommand < Command
     def run
       ids = []
       @argv.map do |host|
-        service.shards_for_hostname(host).each do |shard|
+        manager.shards_for_hostname(host).each do |shard|
           ids << shard.id
         end
       end
@@ -460,11 +457,11 @@ module Gizzard
       displayed = {}
       overlaps.sort_by { |hosts, count| count }.reverse.each do |(host_a, host_b), count|
         next if !host_a || !host_b || displayed[host_a] || displayed[host_b]
-        id_a = ids_by_host[host_a].find { |id| service.list_upward_links(id).size > 0 }
-        id_b = ids_by_host[host_b].find { |id| service.list_upward_links(id).size > 0 }
+        id_a = ids_by_host[host_a].find {|id| manager.list_upward_links(id).size > 0 }
+        id_b = ids_by_host[host_b].find {|id| manager.list_upward_links(id).size > 0 }
         next unless id_a && id_b
-        weight_a = service.list_upward_links(id_a).first.weight
-        weight_b = service.list_upward_links(id_b).first.weight
+        weight_a = manager.list_upward_links(id_a).first.weight
+        weight_b = manager.list_upward_links(id_b).first.weight
         if weight_a > weight_b
           puts "#{host_a}\t#{host_b}"
         else
@@ -483,7 +480,7 @@ module Gizzard
     end
   end
 
-  class ReportCommand < ShardCommand
+  class ReportCommand < Command
     def run
       things = @argv.map do |shard|
         parse(down(ShardId.parse(shard))).join("\n")
@@ -532,7 +529,7 @@ module Gizzard
     end
 
     def down(id)
-      vals = service.list_downward_links(id).map do |link|
+      vals = manager.list_downward_links(id).map do |link|
         down(link.down_id)
       end
       { id.to_unix => vals }
@@ -550,11 +547,11 @@ module Gizzard
     end
   end
 
-  class FindCommand < ShardCommand
+  class FindCommand < Command
     def run
       hosts = @argv << command_options.shard_host
       hosts.compact.each do |host|
-        service.shards_for_hostname(host).each do |shard|
+        manager.shards_for_hostname(host).each do |shard|
           next if command_options.shard_type && shard.class_name !~ Regexp.new(command_options.shard_type)
           output shard.id.to_unix
         end
@@ -562,7 +559,7 @@ module Gizzard
     end
   end
 
-  class LookupCommand < ShardCommand
+  class LookupCommand < Command
     def run
       table_id, source = @argv
       help!("Requires table id and source") unless table_id && source
@@ -572,51 +569,51 @@ module Gizzard
       else
         source_id = source.to_i
       end
-      shard = service.find_current_forwarding(table_id.to_i, source_id)
+      shard = manager.find_current_forwarding(table_id.to_i, source_id)
       output shard.id.to_unix
     end
   end
 
-  class CopyCommand < ShardCommand
+  class CopyCommand < Command
     def run
       from_shard_id_string, to_shard_id_string = @argv
       help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
       from_shard_id = ShardId.parse(from_shard_id_string)
       to_shard_id = ShardId.parse(to_shard_id_string)
-      service.copy_shard(from_shard_id, to_shard_id)
+      manager.copy_shard(from_shard_id, to_shard_id)
     end
   end
 
-  class BusyCommand < ShardCommand
+  class BusyCommand < Command
     def run
-      service.get_busy_shards().each { |shard_info| output shard_info.to_unix }
+      manager.get_busy_shards().each { |shard_info| output shard_info.to_unix }
     end
   end
 
-  class SetupReplicaCommand < ShardCommand
+  class SetupReplicaCommand < Command
     def run
       from_shard_id_string, to_shard_id_string = @argv
       help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
       from_shard_id = ShardId.parse(from_shard_id_string)
       to_shard_id = ShardId.parse(to_shard_id_string)
 
-      if service.list_upward_links(to_shard_id).size > 0
+      if manager.list_upward_links(to_shard_id).size > 0
         STDERR.puts "Destination shard #{to_shard_id} has links to it."
         exit 1
       end
 
-      link = service.list_upward_links(from_shard_id)[0]
+      link = manager.list_upward_links(from_shard_id)[0]
       replica_shard_id = link.up_id
       weight = link.weight
       write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
-      service.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
-      service.add_link(replica_shard_id, write_only_shard_id, weight)
-      service.add_link(write_only_shard_id, to_shard_id, 1)
+      manager.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
+      manager.add_link(replica_shard_id, write_only_shard_id, weight)
+      manager.add_link(write_only_shard_id, to_shard_id, 1)
       output to_shard_id.to_unix
     end
   end
 
-  class FinishReplicaCommand < ShardCommand
+  class FinishReplicaCommand < Command
     def run
       from_shard_id_string, to_shard_id_string = @argv
       help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
@@ -624,58 +621,58 @@ module Gizzard
       to_shard_id = ShardId.parse(to_shard_id_string)
 
       write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
-      link = service.list_upward_links(write_only_shard_id)[0]
+      link = manager.list_upward_links(write_only_shard_id)[0]
       replica_shard_id = link.up_id
       weight = link.weight
 
       # careful. need to validate some basic assumptions.
       unless global_options.force
-        if service.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
+        if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
           STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
           exit 1
         end
-        if service.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
+        if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
           STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
           exit 1
         end
       end
 
-      service.remove_link(write_only_shard_id, to_shard_id)
-      service.remove_link(replica_shard_id, write_only_shard_id)
-      service.add_link(replica_shard_id, to_shard_id, weight)
-      service.delete_shard(write_only_shard_id)
+      manager.remove_link(write_only_shard_id, to_shard_id)
+      manager.remove_link(replica_shard_id, write_only_shard_id)
+      manager.add_link(replica_shard_id, to_shard_id, weight)
+      manager.delete_shard(write_only_shard_id)
     end
   end
 
-  class SetupMigrateCommand < ShardCommand
+  class SetupMigrateCommand < Command
     def run
       from_shard_id_string, to_shard_id_string = @argv
       help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
       from_shard_id = ShardId.parse(from_shard_id_string)
       to_shard_id = ShardId.parse(to_shard_id_string)
 
-      if service.list_upward_links(to_shard_id).size > 0
+      if manager.list_upward_links(to_shard_id).size > 0
         STDERR.puts "Destination shard #{to_shard_id} has links to it."
         exit 1
       end
 
       write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_write_only")
       replica_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_migrate_replica")
-      service.create_shard(ShardInfo.new(write_only_shard_id, "com.twitter.gizzard.shards.WriteOnlyShard", "", "", 0))
-      service.create_shard(ShardInfo.new(replica_shard_id, "com.twitter.gizzard.shards.ReplicatingShard", "", "", 0))
-      service.add_link(write_only_shard_id, to_shard_id, 1)
-      service.list_upward_links(from_shard_id).each do |link|
-        service.remove_link(link.up_id, link.down_id)
-        service.add_link(link.up_id, replica_shard_id, link.weight)
-      end
-      service.add_link(replica_shard_id, from_shard_id, 1)
-      service.add_link(replica_shard_id, write_only_shard_id, 0)
-      service.replace_forwarding(from_shard_id, replica_shard_id)
+      manager.create_shard(ShardInfo.new(write_only_shard_id, "com.twitter.gizzard.shards.WriteOnlyShard", "", "", 0))
+      manager.create_shard(ShardInfo.new(replica_shard_id, "com.twitter.gizzard.shards.ReplicatingShard", "", "", 0))
+      manager.add_link(write_only_shard_id, to_shard_id, 1)
+      manager.list_upward_links(from_shard_id).each do |link|
+        manager.remove_link(link.up_id, link.down_id)
+        manager.add_link(link.up_id, replica_shard_id, link.weight)
+      end
+      manager.add_link(replica_shard_id, from_shard_id, 1)
+      manager.add_link(replica_shard_id, write_only_shard_id, 0)
+      manager.replace_forwarding(from_shard_id, replica_shard_id)
       output replica_shard_id.to_unix
     end
   end
 
-  class FinishMigrateCommand < ShardCommand
+  class FinishMigrateCommand < Command
     def run
       from_shard_id_string, to_shard_id_string = @argv
       help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
@@ -687,57 +684,194 @@ module Gizzard
 
       # careful. need to validate some basic assumptions.
       unless global_options.force
-        if service.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
+        if manager.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
           STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
           exit 1
         end
-        if service.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
+        if manager.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
           STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
           exit 1
         end
-        if service.list_upward_links(write_only_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
+        if manager.list_upward_links(write_only_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
           STDERR.puts "Uplink from write-only barrier is not a migration replica."
           exit 1
         end
       end
 
-      service.remove_link(write_only_shard_id, to_shard_id)
-      service.list_upward_links(replica_shard_id).each do |link|
-        service.remove_link(link.up_id, link.down_id)
-        service.add_link(link.up_id, to_shard_id, link.weight)
+      manager.remove_link(write_only_shard_id, to_shard_id)
+      manager.list_upward_links(replica_shard_id).each do |link|
+        manager.remove_link(link.up_id, link.down_id)
+        manager.add_link(link.up_id, to_shard_id, link.weight)
       end
-      service.replace_forwarding(replica_shard_id, to_shard_id)
-      service.delete_shard(replica_shard_id)
-      service.delete_shard(write_only_shard_id)
+      manager.replace_forwarding(replica_shard_id, to_shard_id)
+      manager.delete_shard(replica_shard_id)
+      manager.delete_shard(write_only_shard_id)
     end
   end
 
-  class InjectCommand < JobCommand
+  class InjectCommand < Command
     def run
+      count     = 0
+      page_size = 20
       priority, *jobs = @argv
       help!("Requires priority") unless priority and jobs.size > 0
-      count = 0
-      jobs.each do |job|
-        service.inject_job(priority.to_i, job)
+
+      jobs.each_slice(page_size) do |js|
+        job_injector.inject_jobs(js.map {|j| Job.new(priority.to_i, j) })
+
         count += 1
         # FIXME add -q --quiet option
         STDERR.print "."
-        STDERR.print "#{count}" if count % 100 == 0
+        STDERR.print "#{count * page_size}" if count % 10 == 0
         STDERR.flush
       end
       STDERR.print "\n"
     end
   end
 
-  class FlushCommand < JobCommand
+  class FlushCommand < Command
     def run
       args = @argv[0]
       help!("Requires --all, or a job priority id.") unless args || command_options.flush_all
       if command_options.flush_all
-        service.retry_errors()
+        manager.retry_errors()
       else
-        service.retry_errors_for(args.to_i)
+        manager.retry_errors_for(args.to_i)
+      end
+    end
+  end
+
+
+  class AddHostCommand < Command
+    def run
+      hosts = @argv.map do |arg|
+        cluster, hostname, port = *arg.split(":")
+        help!("malformed host argument") unless [cluster, hostname, port].compact.length == 3
+
+        Host.new(hostname, port.to_i, cluster, HostStatus::Normal)
       end
+
+      hosts.each {|h| manager.add_remote_host(h) }
+    end
+  end
+
+  class RemoveHostCommand < Command
+    def run
+      host = @argv[0].split(":")
+      host.unshift nil if host.length == 2
+      cluster, hostname, port = *host
+
+      manager.remove_remote_host(hostname, port.to_i)
+    end
+  end
+
+  class ListHostsCommand < Command
+    def run
+      manager.list_remote_hosts.each do |host|
+        puts "#{[host.cluster, host.hostname, host.port].join(":")} #{host.status}"
+      end
+    end
+  end
+
+  class TopologyCommand < Command
+    def run
+      templates = manager.manifest(*global_options.tables).templates.inject({}) do |h, (t, fs)|
+        h.update t.to_config => fs
+      end
+
+      if command_options.forwardings
+        templates.
+          inject([]) { |h, (t, fs)| fs.each { |f| h << [f.base_id, t] }; h }.
+          sort.
+          each { |a| puts "%25d\t%s" % a }
+      else
+        templates.
+          map { |(t, fs)| [fs.length, t] }.
+          sort.reverse.
+          each { |a| puts "%4d %s" % a }
+      end
+    end
+  end
+
+  class TransformTreeCommand < Command
+    def run
+      help!("wrong number of arguments") unless @argv.length == 2
+
+      scheduler_options = command_options.scheduler_options || {}
+      template_s, shard_id_s = @argv
+
+      to_template    = ShardTemplate.parse(template_s)
+      shard_id       = ShardId.parse(shard_id_s)
+      base_name      = shard_id.table_prefix.split('_').first
+      forwarding     = manager.get_forwarding_for_shard(shard_id)
+      manifest       = manager.manifest(forwarding.table_id)
+      shard          = manifest.trees[forwarding]
+      copy_wrapper   = scheduler_options[:copy_wrapper]
+      be_quiet       = global_options.force && command_options.quiet
+      transformation = Transformation.new(shard.template, to_template, copy_wrapper)
+
+      scheduler_options[:quiet] = be_quiet
+
+      unless be_quiet
+        puts transformation.inspect
+        puts ""
+      end
+
+      unless global_options.force
+        print "Continue? (y/n) "; $stdout.flush
+        exit unless $stdin.gets.chomp == "y"
+        puts ""
+      end
+
+      Gizzard.schedule! manager,
+                        base_name,
+                        { transformation => { forwarding => shard } },
+                        scheduler_options
+    end
+  end
+
+  class TransformCommand < Command
+    def run
+      help!("must have an even number of arguments") unless @argv.length % 2 == 0
+
+      scheduler_options = command_options.scheduler_options || {}
+      manifest          = manager.manifest(*global_options.tables)
+      copy_wrapper      = scheduler_options[:copy_wrapper]
+      be_quiet          = global_options.force && command_options.quiet
+      transformations   = {}
+
+      scheduler_options[:quiet] = be_quiet
+
+      @argv.each_slice(2) do |(from_template_s, to_template_s)|
+        from, to       = [from_template_s, to_template_s].map {|s| ShardTemplate.parse(s) }
+        transformation = Transformation.new(from, to, copy_wrapper)
+        forwardings    = Set.new(manifest.templates[from] || [])
+        trees          = manifest.trees.reject {|(f, s)| !forwardings.include?(f) }
+
+        transformations[transformation] = trees
+      end
+
+      base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
+
+      unless be_quiet
+        transformations.each do |transformation, trees|
+          puts transformation.inspect
+          puts "Applied to #{trees.length} shards:"
+          trees.keys.sort.each {|f| puts "  #{f.inspect}" }
+        end
+        puts ""
+      end
+
+      unless global_options.force
+        print "Continue? (y/n) "; $stdout.flush
+        exit unless $stdin.gets.chomp == "y"
+        puts ""
+      end
+
+      Gizzard.schedule! manager,
+                        base_name,
+                        transformations,
+                        scheduler_options
     end
   end
 end