gizzmo 0.11.0 → 0.11.1

data/lib/gizzard/transformation_op.rb

@@ -0,0 +1,181 @@
+module Gizzard
+  module Transformation::Op
+    class BaseOp
+      def inverse?(other)
+        Transformation::OP_INVERSES[self.class] == other.class
+      end
+
+      def eql?(other)
+        self.class == other.class
+      end
+
+      alias == eql?
+
+      def inspect
+        templates = (is_a?(LinkOp) ? [from, to] : [template]).map {|t| t.identifier }.join(" -> ")
+        name      = Transformation::OP_NAMES[self.class]
+        "#{name}(#{templates})"
+      end
+
+      def <=>(other)
+        Transformation::OP_PRIORITIES[self.class] <=> Transformation::OP_PRIORITIES[other.class]
+      end
+
+      def involved_shards(table_prefix, translations)
+        []
+      end
+    end
+
+    class CopyShard < BaseOp
+      BUSY = 1
+
+      attr_reader :from, :to
+      alias template to
+
+      def initialize(from, to)
+        @from = from
+        @to   = to
+      end
+
+      def expand(*args); { :copy => [self] } end
+
+      def involved_shards(table_prefix, translations)
+        [to.to_shard_id(table_prefix, translations)]
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        from_shard_id = from.to_shard_id(table_prefix, translations)
+        to_shard_id   = to.to_shard_id(table_prefix, translations)
+
+        nameserver.mark_shard_busy(to_shard_id, BUSY)
+        nameserver.copy_shard(from_shard_id, to_shard_id)
+      end
+    end
+
+    class LinkOp < BaseOp
+      attr_reader :from, :to
+      alias template to
+
+      def initialize(from, to)
+        @from = from
+        @to   = to
+      end
+
+      def inverse?(other)
+        super && self.from.link_eql?(other.from) && self.to.link_eql?(other.to)
+      end
+
+      def eql?(other)
+        super && self.from.link_eql?(other.from) && self.to.link_eql?(other.to)
+      end
+    end
+
+    class AddLink < LinkOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        if involved_in_copy
+          wrapper = ShardTemplate.new(wrapper_type, to.host, to.weight, '', '', [to])
+          { :prepare => [AddLink.new(from, wrapper)],
+            :cleanup => [self, RemoveLink.new(from, wrapper)] }
+        else
+          { :prepare => [self] }
+        end
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        from_shard_id = from.to_shard_id(table_prefix, translations)
+        to_shard_id   = to.to_shard_id(table_prefix, translations)
+
+        nameserver.add_link(from_shard_id, to_shard_id, to.weight)
+      end
+    end
+
+    class RemoveLink < LinkOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        { (involved_in_copy ? :cleanup : :prepare) => [self] }
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        from_shard_id = from.to_shard_id(table_prefix, translations)
+        to_shard_id   = to.to_shard_id(table_prefix, translations)
+
+        nameserver.remove_link(from_shard_id, to_shard_id)
+      end
+    end
+
+    class ShardOp < BaseOp
+      attr_reader :template
+
+      def initialize(template)
+        @template = template
+      end
+
+      def inverse?(other)
+        super && self.template.shard_eql?(other.template)
+      end
+
+      def eql?(other)
+        super && self.template.shard_eql?(other.template)
+      end
+    end
+
+    class CreateShard < ShardOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        if involved_in_copy
+          wrapper = ShardTemplate.new(wrapper_type, template.host, template.weight, '', '', [template])
+          { :prepare => [self, CreateShard.new(wrapper), AddLink.new(wrapper, template)],
+            :cleanup => [RemoveLink.new(wrapper, template), DeleteShard.new(wrapper)],
+            :copy => [CopyShard.new(copy_source, template)] }
+        else
+          { :prepare => [self] }
+        end
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        nameserver.create_shard(template.to_shard_info(table_prefix, translations))
+      end
+    end
+
+    class DeleteShard < ShardOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        { (involved_in_copy ? :cleanup : :prepare) => [self] }
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        nameserver.delete_shard(template.to_shard_id(table_prefix, translations))
+      end
+    end
+
+    class SetForwarding < ShardOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        if involved_in_copy
+          wrapper = ShardTemplate.new(wrapper_type, nil, 0, '', '', [to])
+          { :prepare => [SetForwarding.new(template, wrapper)],
+            :cleanup => [self] }
+        else
+          { :prepare => [self] }
+        end
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        shard_id   = template.to_shard_id(table_prefix, translations)
+        forwarding = Forwarding.new(table_id, base_id, shard_id)
+        nameserver.set_forwarding(forwarding)
+      end
+    end
+
+
+    # XXX: A no-op, but needed for setup/teardown symmetry
+
+    class RemoveForwarding < ShardOp
+      def expand(copy_source, involved_in_copy, wrapper_type)
+        { (involved_in_copy ? :cleanup : :prepare) => [self] }
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        # shard_id   = template.to_shard_id(table_prefix, translations)
+        # forwarding = Forwarding.new(table_id, base_id, shard_id)
+        # nameserver.remove_forwarding(forwarding)
+      end
+    end
+  end
+end

data/lib/gizzard/transformation_scheduler.rb

@@ -0,0 +1,220 @@
+module Gizzard
+  def self.schedule!(*args)
+    Transformation::Scheduler.new(*args).apply!
+  end
+
+  class Transformation::Scheduler
+
+    attr_reader :nameserver, :transformations
+    attr_reader :max_copies, :copies_per_host
+
+    DEFAULT_OPTIONS = {
+      :max_copies      => 30,
+      :copies_per_host => 8,
+      :poll_interval   => 10
+    }.freeze
+
+    def initialize(nameserver, base_name, transformations, options = {})
+      options = DEFAULT_OPTIONS.merge(options)
+      @nameserver         = nameserver
+      @transformations    = transformations
+      @max_copies         = options[:max_copies]
+      @copies_per_host    = options[:copies_per_host]
+      @poll_interval      = options[:poll_interval]
+      @be_quiet           = options[:quiet]
+      @dont_show_progress = options[:no_progress] || @be_quiet
+
+      @jobs_in_progress = []
+      @jobs_finished    = []
+
+      @jobs_pending = transformations.map do |transformation, forwardings_to_shards|
+        transformation.bind(base_name, forwardings_to_shards)
+      end.flatten
+    end
+
+    # to schedule a job:
+    # 1. pull a job that does not involve a disqualified host.
+    # 2. run prepare ops
+    # 3. reload app servers
+    # 4. schedule copy
+    # 5. put in jobs_in_progress
+
+    # on job completion:
+    # 1. run cleanup ops
+    # 2. remove from jobs_in_progress
+    # 3. put in jos_finished
+    # 4. schedule a new job or reload app servers.
+
+    def apply!
+      @start_time = Time.now
+
+      loop do
+        reload_busy_shards
+
+        cleanup_jobs
+        schedule_jobs(max_copies - busy_shards.length)
+
+        break if @jobs_pending.empty? && @jobs_in_progress.empty?
+
+        unless nameserver.dryrun?
+          if @dont_show_progress
+            sleep(@poll_interval)
+          else
+            sleep_with_progress(@poll_interval)
+          end
+        end
+      end
+
+      nameserver.reload_config
+
+      log "#{@jobs_finished.length} transformation#{'s' if @jobs_finished.length > 1} applied. Total time elapsed: #{time_elapsed}"
+    end
+
+    def schedule_jobs(num_to_schedule)
+      to_be_busy_hosts = []
+
+      jobs = (1..num_to_schedule).map do
+        job = @jobs_pending.find do |j|
+          (busy_hosts(to_be_busy_hosts) & j.involved_hosts).empty?
+        end
+
+        if job
+          to_be_busy_hosts.concat job.involved_hosts
+          @jobs_pending.delete(job)
+        end
+
+        job
+      end.compact.sort_by {|t| t.forwarding }
+
+      unless jobs.empty?
+        log "STARTING:"
+        jobs.each do |j|
+          log "  #{j.inspect}"
+          j.prepare!(nameserver)
+        end
+
+        nameserver.reload_config
+
+        copy_jobs = jobs.select {|j| j.copy_required? }
+
+        unless copy_jobs.empty?
+          log "COPIES:"
+          copy_jobs.each do |j|
+            j.copy_descs.each {|d| log "  #{d}" }
+            j.copy!(nameserver)
+          end
+
+          reload_busy_shards
+        end
+
+        @jobs_in_progress.concat(jobs)
+      end
+    end
+
+    def cleanup_jobs
+      jobs = jobs_completed
+
+      unless jobs.empty?
+        @jobs_in_progress -= jobs
+
+        log "FINISHING:"
+        jobs.each do |j|
+          log "  #{j.inspect}"
+          j.cleanup!(nameserver)
+        end
+
+        @jobs_finished.concat(jobs)
+      end
+    end
+
+    def jobs_completed
+      @jobs_in_progress.select {|j| (busy_shards & j.involved_shards).empty? }
+    end
+
+    def reload_busy_shards
+      @busy_shards = nil
+      busy_shards
+    end
+
+    def busy_shards
+      @busy_shards ||=
+        if nameserver.dryrun?
+          []
+        else
+          nameserver.get_busy_shards.map {|s| s.id }
+        end
+    end
+
+    def busy_hosts(extra_hosts = [])
+      hosts = extra_hosts + busy_shards.map {|s| s.hostname }
+
+      copies_count_map = hosts.inject({}) do |h, host|
+        h.update(host => 1) {|_,a,b| a + b }
+      end
+
+      copies_count_map.select {|_, count| count >= @copies_per_host }.map {|(host, _)| host }
+    end
+
+    def sleep_with_progress(interval)
+      start = Time.now
+      while (Time.now - start) < interval
+        put_copy_progress
+        sleep 0.2
+      end
+    end
+
+    def clear_progress_string
+      if @progress_string
+        print "\r" + (" " * (@progress_string.length + 10)) + "\r"
+        @progress_string = nil
+      end
+    end
+
+    def log(*args)
+      unless @be_quiet
+        clear_progress_string
+        puts *args
+      end
+    end
+
+    def put_copy_progress
+      @i ||= 0
+      @i  += 1
+
+      unless @jobs_in_progress.empty? || busy_shards.empty?
+        spinner         = ['-', '\\', '|', '/'][@i % 4]
+        elapsed_txt     = "Time elapsed: #{time_elapsed}"
+        pending_txt     = "Pending: #{@jobs_pending.length}"
+        finished_txt    = "Finished: #{@jobs_finished.length}"
+        in_progress_txt =
+          if busy_shards.length != @jobs_in_progress.length
+            "In progress: #{@jobs_in_progress.length} (Copies: #{busy_shards.length})"
+          else
+            "In progress: #{@jobs_in_progress.length}"
+          end
+
+        clear_progress_string
+
+        @progress_string = "#{spinner} #{in_progress_txt} #{pending_txt} #{finished_txt} #{elapsed_txt}"
+        print @progress_string; $stdout.flush
+      end
+    end
+
+    def time_elapsed
+      s = (Time.now - @start_time).to_i
+
+      if s == 1
+        "1 second"
+      elsif s < 60
+        "#{s} seconds"
+      else
+        days    = s / (60 * 60 * 24)               if s >= 60 * 60 * 24
+        hours   = (s % (60 * 60 * 24)) / (60 * 60) if s >= 60 * 60
+        minutes = (s % (60 * 60)) / 60             if s >= 60
+        seconds = s % 60
+
+        [days,hours,minutes,seconds].compact.map {|i| "%0.2i" % i }.join(":")
+      end
+    end
+  end
+end

data/lib/gizzmo.rb

@@ -22,7 +22,7 @@ DOC_STRINGS = {
   "lookup" => "Lookup the shard id that holds the record for a given table / source_id.",
   "markbusy" => "Mark a shard as busy.",
   "pair" => "Report the replica pairing structure for a list of hosts.",
-  "reload" => "Instruct an appserver to reload its nameserver state.",
+  "reload" => "Instruct application servers to reload the nameserver state.",
   "report" => "Show each unique replica structure for a given list of shards. Usually this shard list comes from << gizzmo forwardings | awk '{ print $3 }' >>.",
   "setup-replica" => "Add a replica to be parallel to an existing replica, in write-only mode, ready to be copied to.",
   "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
@@ -32,9 +32,12 @@ ORIGINAL_ARGV = ARGV.dup
 zero = File.basename($0)
 
 # Container for parsed options
-global_options     = OpenStruct.new
-global_options.render = []
-global_options.framed = false
+global_options = OpenStruct.new
+global_options.port          = 7920
+global_options.injector_port = 7921
+global_options.render        = []
+global_options.framed        = false
+
 subcommand_options = OpenStruct.new
 
 # Leftover arguments
@@ -78,10 +81,29 @@ end
 
 def load_config(options, filename)
   YAML.load(File.open(filename)).each do |k, v|
+    v = v.split(",").map {|h| h.strip } if k == "hosts"
     options.send("#{k}=", v)
   end
 end
 
+def add_scheduler_opts(subcommand_options, opts)
+  opts.on("--max-copies=COUNT", "Limit max simultaneous copies to COUNT.") do |c|
+    (subcommand_options.scheduler_options ||= {})[:max_copies] = c.to_i
+  end
+  opts.on("--copies-per-host=COUNT", "Limit max copies per individual destination host to COUNT") do |c|
+    (subcommand_options.scheduler_options ||= {})[:copies_per_host] = c.to_i
+  end
+  opts.on("--poll-interval=SECONDS", "Sleep SECONDS between polling for copy status") do |c|
+    (subcommand_options.scheduler_options ||= {})[:poll_interval] = c.to_i
+  end
+  opts.on("--copy-wrapper=TYPE", "Wrap copy destination shards with TYPE. default WriteOnlyShard") do |t|
+    (subcommand_options.scheduler_options ||= {})[:copy_wrapper] = t
+  end
+  opts.on("--no-progress", "Do not show progress bar at bottom.") do
+    (subcommand_options.scheduler_options ||= {})[:no_progress] = true
+  end
+end
+
 subcommands = {
   'create' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} create [options] CLASS_NAME SHARD_ID [MORE SHARD_IDS...]"
@@ -106,7 +128,7 @@ subcommands = {
     opts.on("-w", "--write-only=CLASS") do |w|
       subcommand_options.write_only_shard = w
     end
-    opts.on("-h", "--hosts=list") do |h|
+    opts.on("-h", "--shard-hosts=list") do |h|
       subcommand_options.hosts = h
     end
     opts.on("-x", "--exclude-hosts=list") do |x|
@@ -149,10 +171,6 @@ subcommands = {
     opts.banner = "Usage: #{zero} addforwarding TABLE_ID BASE_ID SHARD_ID"
     separators(opts, DOC_STRINGS["addforwarding"])
   end,
-  'currentforwarding' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} currentforwarding SOURCE_ID [ANOTHER_SOURCE_ID...]"
-    separators(opts, DOC_STRINGS["addforwarding"])
-  end,
   'forwardings' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} forwardings [options]"
     separators(opts, DOC_STRINGS["forwardings"])
@@ -177,7 +195,7 @@ subcommands = {
       subcommand_options.shard_type = shard_type
     end
 
-    opts.on("-H", "--host=HOST", "HOST of shard") do |shard_host|
+    opts.on("-h", "--shard-host=HOST", "HOST of shard") do |shard_host|
       subcommand_options.shard_host = shard_host
     end
   end,
@@ -267,6 +285,46 @@ subcommands = {
     opts.on("--all", "Flush all error queues.") do
       subcommand_options.flush_all = true
     end
+  end,
+  'add-host' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} add-host HOSTS"
+    separators(opts, DOC_STRINGS["add-host"])
+  end,
+  'remove-host' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} remove-host HOST"
+    separators(opts, DOC_STRINGS["remove-host"])
+  end,
+  'list-hosts' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} list-hosts"
+    separators(opts, DOC_STRINGS["list-hosts"])
+  end,
+  'topology' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} topology [options]"
+    separators(opts, DOC_STRINGS["topology"])
+
+    opts.on("--forwardings", "Show topology of forwardings instead of counts") do
+      subcommand_options.forwardings = true
+    end
+  end,
+  'transform-tree' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} transform-tree [options] ROOT_SHARD_ID TEMPLATE"
+    separators(opts, DOC_STRINGS['transform-tree'])
+
+    add_scheduler_opts subcommand_options, opts
+
+    opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
+  end,
+  'transform' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} transform [options] FROM_TEMPLATE TO_TEMPLATE"
+    separators(opts, DOC_STRINGS['transform'])
+
+    add_scheduler_opts subcommand_options, opts
+
+    opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
   end
 }
 
@@ -288,7 +346,7 @@ global = OptionParser.new do |opts|
   opts.separator "key/value pairs corresponding to options you want by default. A common .gizzmorc"
   opts.separator "simply contains:"
   opts.separator ""
-  opts.separator "    host: localhost"
+  opts.separator "    hosts: localhost"
   opts.separator "    port: 7917"
   opts.separator ""
   opts.separator "Subcommands:"
@@ -305,24 +363,32 @@ global = OptionParser.new do |opts|
   opts.separator ""
   opts.separator ""
   opts.separator "Global options:"
-  opts.on("-H", "--host=HOSTNAME", "HOSTNAME of remote thrift service") do |host|
-    global_options.host = host
+  opts.on("-H", "--hosts=HOST[,HOST,...]", "HOSTS of application servers") do |hosts|
+    global_options.hosts = hosts.split(",").map {|h| h.strip }
   end
 
-  opts.on("-P", "--port=PORT", "PORT of remote thrift service") do |port|
+  opts.on("-P", "--port=PORT", "PORT of remote manager service. default 7920") do |port|
     global_options.port = port.to_i
   end
 
+  opts.on("-I", "--injector=PORT", "PORT of remote job injector service. default 7921") do |port|
+    global_options.injector_port = port.to_i
+  end
+
+  opts.on("-T", "--tables=TABLE[,TABLE,...]", "TABLE ids of forwardings to affect") do |tables|
+    global_options.tables = tables.split(",").map {|t| t.to_i }
+  end
+
   opts.on("-F", "--framed", "use the thrift framed transport") do |framed|
     global_options.framed = true
   end
 
   opts.on("-r", "--retry=TIMES", "TIMES to retry the command") do |r|
-    global_options.retry = r
+    global_options.retry = r.to_i
   end
 
   opts.on("-t", "--timeout=SECONDS", "SECONDS to let the command run") do |r|
-    global_options.timeout = r
+    global_options.timeout = r.to_i
   end
 
   opts.on("--subtree", "Render in subtree mode") do
@@ -333,7 +399,7 @@ global = OptionParser.new do |opts|
     global_options.render << "info"
   end
 
-  opts.on("-D", "--dry-run", "") do |port|
+  opts.on("-D", "--dry-run", "") do
     global_options.dry = true
   end
 
@@ -401,12 +467,12 @@ def custom_timeout(seconds)
     begin
       require "rubygems"
       require "system_timer"
-      SystemTimer.timeout_after(seconds.to_i) do
+      SystemTimer.timeout_after(seconds) do
         yield
       end
     rescue LoadError
       require "timeout"
-      Timeout.timeout(seconds.to_i) do
+      Timeout.timeout(seconds) do
         yield
       end
     end
@@ -427,8 +493,9 @@ rescue HelpNeededError => e
   end
   STDERR.puts subcommands[subcommand_name]
   exit 1
-rescue ThriftClient::Simple::ThriftException, Gizzard::Thrift::ShardException, Errno::ECONNREFUSED => e
+rescue ThriftClient::Simple::ThriftException, Gizzard::GizzardException, Errno::ECONNREFUSED => e
   STDERR.puts e.message
+  STDERR.puts e.backtrace
   exit 1
 rescue Errno::EPIPE
   # This is just us trying to puts into a closed stdout.  For example, if you pipe into