gizzmo 0.11.0 → 0.11.1

data/lib/gizzard/thrift.rb

@@ -1,177 +1,229 @@
-#!/usr/bin/env ruby
 require 'vendor/thrift_client/simple'
 
 module Gizzard
-  module Thrift
-    T = ThriftClient::Simple
+  T = ThriftClient::Simple
 
-    def self.struct(*args)
-      T::StructType.new(*args)
+  def self.struct(*args)
+    T::StructType.new(*args)
+  end
+
+  def self.list(*args)
+    T::ListType.new(*args)
+  end
+
+  GizzardException = T.make_exception(:GizzardException,
+    T::Field.new(:description, T::STRING, 1)
+  )
+
+  ShardId = T.make_struct(:ShardId,
+    T::Field.new(:hostname, T::STRING, 1),
+    T::Field.new(:table_prefix, T::STRING, 2)
+  )
+
+  class ShardId
+    def inspect
+      "#{hostname}/#{table_prefix}"
     end
 
-    ShardException = T.make_exception(:ShardException,
-      T::Field.new(:description, T::STRING, 1)
-    )
+    def <=>(o)
+      self.hostname <=> o.hostname
+    end
 
-    ShardId = T.make_struct(:ShardId,
-      T::Field.new(:hostname, T::STRING, 1),
-      T::Field.new(:table_prefix, T::STRING, 2)
-    )
+    alias_method :to_unix, :inspect
 
-    class ShardId
-      def inspect
-        "#{hostname}/#{table_prefix}"
-      end
+    def self.parse(string)
+      new(*string.match("(.*)/(.*)").values_at(1, 2))
+    end
+  end
 
-      def <=>(o)
-        self.hostname <=> o.hostname
-      end
+  ShardInfo = T.make_struct(:ShardInfo,
+    T::Field.new(:id, struct(ShardId), 1),
+    T::Field.new(:class_name, T::STRING, 2),
+    T::Field.new(:source_type, T::STRING, 3),
+    T::Field.new(:destination_type, T::STRING, 4),
+    T::Field.new(:busy, T::I32, 5)
+  )
+
+  class ShardInfo
+    def busy?
+      busy && busy > 0
+    end
 
-      alias_method :to_unix, :inspect
+    def inspect(short = false)
+      "#{id.inspect}" + (busy? ? " (BUSY)" : "")
+    end
 
-      def self.parse(string)
-        new(*string.split("/"))
-      end
+    def to_unix
+      [id.to_unix, class_name, busy? ? "busy" : "ok"].join("\t")
     end
+  end
 
-    ShardInfo = T.make_struct(:ShardInfo,
-      T::Field.new(:id, struct(ShardId), 1),
-      T::Field.new(:class_name, T::STRING, 2),
-      T::Field.new(:source_type, T::STRING, 3),
-      T::Field.new(:destination_type, T::STRING, 4),
-      T::Field.new(:busy, T::I32, 5)
-    )
-
-    class ShardInfo
-      def busy?
-        busy && busy > 0
-      end
+  LinkInfo = T.make_struct(:LinkInfo,
+    T::Field.new(:up_id, struct(ShardId), 1),
+    T::Field.new(:down_id, struct(ShardId), 2),
+    T::Field.new(:weight, T::I32, 3)
+  )
 
-      def inspect(short = false)
-        "#{id.inspect}" + (busy? ? " (BUSY)" : "")
-      end
+  class LinkInfo
+    def inspect
+      "#{up_id.inspect} -> #{down_id.inspect}" + (weight == 1 ? "" : " <#{weight}>")
+    end
 
-      def to_unix
-        [id.to_unix, class_name, busy? ? "busy" : "ok"].join("\t")
-      end
+    def to_unix
+      [up_id.to_unix, down_id.to_unix, weight].join("\t")
     end
+  end
 
-    LinkInfo = T.make_struct(:LinkInfo,
-      T::Field.new(:up_id, struct(ShardId), 1),
-      T::Field.new(:down_id, struct(ShardId), 2),
-      T::Field.new(:weight, T::I32, 3)
-    )
+  Forwarding = T.make_struct(:Forwarding,
+    T::Field.new(:table_id, T::I32, 1),
+    T::Field.new(:base_id, T::I64, 2),
+    T::Field.new(:shard_id, struct(ShardId), 3)
+  )
 
-    class LinkInfo
-      def inspect
-        "#{up_id.inspect} -> #{down_id.inspect}" + (weight == 1 ? "" : " <#{weight}>")
-      end
+  class Forwarding
+    def <=>(o)
+      [self.table_id, self.base_id] <=> [o.table_id, o.base_id]
+    end
 
-      def to_unix
-        [up_id.to_unix, down_id.to_unix, weight].join("\t")
-      end
+    def inspect
+      "[#{table_id}] #{base_id.to_s(16)} = #{shard_id.inspect}"
     end
+  end
 
-    ShardMigration = T.make_struct(:ShardMigration,
-      T::Field.new(:source_id, struct(ShardId), 1),
-      T::Field.new(:destination_id, struct(ShardId), 2)
-    )
-
-    Forwarding = T.make_struct(:Forwarding,
-      T::Field.new(:table_id, T::I32, 1),
-      T::Field.new(:base_id, T::I64, 2),
-      T::Field.new(:shard_id, struct(ShardId), 3)
-    )
-
-    class Forwarding
-      #FIXME table_id is not human-readable
-      def inspect
-        "[#{table_id}] #{base_id.to_s(16)} -> #{shard_id.inspect}"
-      end
+  NameServerState = T.make_struct(:NameserverState,
+    T::Field.new(:shards, list(struct(ShardInfo)), 1),
+    T::Field.new(:links, list(struct(LinkInfo)), 2),
+    T::Field.new(:forwardings, list(struct(Forwarding)), 3),
+    T::Field.new(:table_id, T::I32, 4)
+  )
+
+
+  module HostStatus
+    Normal  = 0
+    Offline = 1
+    Blocked = 2
+  end
+
+  Host = T.make_struct(:Host,
+    T::Field.new(:hostname, T::STRING, 1),
+    T::Field.new(:port, T::I32, 2),
+    T::Field.new(:cluster, T::STRING, 3),
+    T::Field.new(:status, T::I32, 4)
+  )
+
+  class Host
+    def inspect
+      "(#{hostname}:#{port} - #{cluster} (#{status})"
     end
+  end
 
-    class GizzmoService < T::ThriftService
-      def initialize(host, port, log_path, framed, dry_run = false)
-        super(host, port, framed)
-        @dry = dry_run
-        begin
-          @log = File.open(log_path, "a")
-        rescue
-          STDERR.puts "Error opening log file at #{log_path}.  Continuing..."
-        end
+  class GizzmoService < T::ThriftService
+    def initialize(host, port, log_path, framed, dry_run = false)
+      super(host, port, framed)
+      @dry = dry_run
+      begin
+        @log = File.open(log_path, "a")
+      rescue
+        STDERR.puts "Error opening log file at #{log_path}.  Continuing..."
       end
+    end
 
-      def _proxy(method_name, *args)
-        cls = self.class.ancestors.find { |cls| cls.respond_to?(:_arg_structs) and cls._arg_structs[method_name.to_sym] }
-        arg_class, rv_class = cls._arg_structs[method_name.to_sym]
-
-        # Writing methods return void. Methods should never both read and write. If this assumption
-        # is violated in the future, dry-run will fail!!
-        is_writing_method = rv_class._fields.first.type == ThriftClient::Simple::VOID
-        if @dry && is_writing_method
-          puts "Skipped writing: #{printable(method_name, args)}"
-        else
-          @log.puts printable(method_name, args, true)
-          super(method_name, *args)
-        end
-      rescue ThriftClient::Simple::ThriftException
-        if @dry
-          puts "Skipped reading: #{printable(method_name, args)}"
-        else
-          raise
-        end
+    def _proxy(method_name, *args)
+      cls = self.class.ancestors.find { |cls| cls.respond_to?(:_arg_structs) and cls._arg_structs[method_name.to_sym] }
+      arg_class, rv_class = cls._arg_structs[method_name.to_sym]
+
+      # Writing methods return void. Methods should never both read and write. If this assumption
+      # is violated in the future, dry-run will fail!!
+      is_writing_method = rv_class._fields.first.type == ThriftClient::Simple::VOID
+      if @dry && is_writing_method
+        puts "Skipped writing: #{printable(method_name, args)}"
+      else
+        @log.puts printable(method_name, args, true)
+        super(method_name, *args)
       end
+    end
 
-      def printable(method_name, args, timestamp = false)
-        ts = timestamp ? "#{Time.now}\t" : ""
-        "#{ts}#{method_name}(#{args.map{|a| a.inspect}.join(', ')})"
-      end
+    def printable(method_name, args, timestamp = false)
+      ts = timestamp ? "#{Time.now}\t" : ""
+      "#{ts}#{method_name}(#{args.map{|a| a.inspect}.join(', ')})"
     end
+  end
 
-    class ShardManager < GizzmoService
-      thrift_method :create_shard, void, field(:shard, struct(ShardInfo), 1), :throws => exception(ShardException)
-      thrift_method :delete_shard, void, field(:id, struct(ShardId), 1), :throws => exception(ShardException)
-      thrift_method :get_shard, struct(ShardInfo), field(:id, struct(ShardId), 1), :throws => exception(ShardException)
+  class Manager < GizzmoService
+    thrift_method :reload_config, void, :throws => exception(GizzardException)
+    thrift_method :rebuild_schema, void, :throws => exception(GizzardException)
 
-      thrift_method :add_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2), field(:weight, i32, 3), :throws => exception(ShardException)
-      thrift_method :remove_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2), :throws => exception(ShardException)
+    thrift_method :find_current_forwarding, struct(ShardInfo), field(:table_id, i32, 1), field(:id, i64, 2), :throws => exception(GizzardException)
 
-      thrift_method :list_upward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1), :throws => exception(ShardException)
-      thrift_method :list_downward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1), :throws => exception(ShardException)
 
-      thrift_method :get_child_shards_of_class, list(struct(ShardInfo)), field(:parent_id, struct(ShardId), 1), field(:class_name, string, 2), :throws => exception(ShardException)
+    # Shard Tree Management
 
-      thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2), :throws => exception(ShardException)
-      thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2), :throws => exception(ShardException)
+    thrift_method :create_shard, void, field(:shard, struct(ShardInfo), 1), :throws => exception(GizzardException)
+    thrift_method :delete_shard, void, field(:id, struct(ShardId), 1), :throws => exception(GizzardException)
 
-      thrift_method :set_forwarding, void, field(:forwarding, struct(Forwarding), 1), :throws => exception(ShardException)
-      thrift_method :replace_forwarding, void, field(:old_id, struct(ShardId), 1), field(:new_id, struct(ShardId), 2), :throws => exception(ShardException)
-      thrift_method :remove_forwarding, void, field(:forwarding, struct(Forwarding), 1), :throws => exception(ShardException)
+    thrift_method :add_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2), field(:weight, i32, 3), :throws => exception(GizzardException)
+    thrift_method :remove_link, void, field(:up_id, struct(ShardId), 1), field(:down_id, struct(ShardId), 2), :throws => exception(GizzardException)
 
-      thrift_method :get_forwarding, struct(Forwarding), field(:table_id, i32, 1), field(:base_id, i64, 2), :throws => exception(ShardException)
-      thrift_method :get_forwarding_for_shard, struct(Forwarding), field(:shard_id, struct(ShardId), 1), :throws => exception(ShardException)
+    thrift_method :set_forwarding, void, field(:forwarding, struct(Forwarding), 1), :throws => exception(GizzardException)
+    thrift_method :replace_forwarding, void, field(:old_id, struct(ShardId), 1), field(:new_id, struct(ShardId), 2), :throws => exception(GizzardException)
+    thrift_method :remove_forwarding, void, field(:forwarding, struct(Forwarding), 1), :throws => exception(GizzardException)
 
-      thrift_method :get_forwardings, list(struct(Forwarding)), :throws => exception(ShardException)
-      thrift_method :reload_forwardings, void, :throws => exception(ShardException)
+    thrift_method :get_shard, struct(ShardInfo), field(:id, struct(ShardId), 1), :throws => exception(GizzardException)
+    thrift_method :shards_for_hostname, list(struct(ShardInfo)), field(:hostname, string, 1), :throws => exception(GizzardException)
+    thrift_method :get_busy_shards, list(struct(ShardInfo)), :throws => exception(GizzardException)
 
-      thrift_method :find_current_forwarding, struct(ShardInfo), field(:table_id, i32, 1), field(:id, i64, 2), :throws => exception(ShardException)
+    thrift_method :list_upward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1), :throws => exception(GizzardException)
+    thrift_method :list_downward_links, list(struct(LinkInfo)), field(:id, struct(ShardId), 1), :throws => exception(GizzardException)
+    thrift_method :get_forwarding, struct(Forwarding), field(:table_id, i32, 1), field(:base_id, i64, 2), :throws => exception(GizzardException)
+    thrift_method :get_forwarding_for_shard, struct(Forwarding), field(:shard_id, struct(ShardId), 1), :throws => exception(GizzardException)
+    thrift_method :get_forwardings, list(struct(Forwarding)), :throws => exception(GizzardException)
 
-      thrift_method :shards_for_hostname, list(struct(ShardInfo)), field(:hostname, string, 1), :throws => exception(ShardException)
-      thrift_method :get_busy_shards, list(struct(ShardInfo)), :throws => exception(ShardException)
-      thrift_method :list_hostnames, list(string), :throws => exception(ShardException)
+    thrift_method :list_hostnames, list(string), :throws => exception(GizzardException)
 
-      thrift_method :rebuild_schema, void, :throws => exception(ShardException)
-    end
+    thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2), :throws => exception(GizzardException)
+    thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2), :throws => exception(GizzardException)
 
-    class JobManager < GizzmoService
-      thrift_method :retry_errors, void
-      thrift_method :stop_writes, void
-      thrift_method :resume_writes, void
-      thrift_method :retry_errors_for, void, field(:priority, i32, 1)
-      thrift_method :stop_writes_for, void, field(:priority, i32, 1)
-      thrift_method :resume_writes_for, void, field(:priority, i32, 1)
-      thrift_method :is_writing, bool, field(:priority, i32, 1)
-      thrift_method :inject_job, void, field(:priority, i32, 1), field(:job, string, 2)
-    end
+    thrift_method :dump_nameserver, struct(NameServerState), field(:table_id, i32, 1), :throws => exception(GizzardException)
+
+
+    # Job Scheduler Management
+
+    thrift_method :retry_errors, void
+    thrift_method :stop_writes, void
+    thrift_method :resume_writes, void
+
+    thrift_method :retry_errors_for, void, field(:priority, i32, 1)
+    thrift_method :stop_writes_for, void, field(:priority, i32, 1)
+    thrift_method :resume_writes_for, void, field(:priority, i32, 1)
+
+    thrift_method :is_writing, bool, field(:priority, i32, 1)
+
+
+    # Remote Host Cluster Management
+
+    thrift_method :add_remote_host, void, field(:host, struct(Host), 1)#, :throws => exception(GizzardException)
+    thrift_method :remove_remote_host, void, field(:hostname, string, 1), field(:port, i32, 2), :throws => exception(GizzardException)
+    thrift_method :set_remote_host_status, void, field(:hostname, string, 1), field(:port, i32, 2), field(:status, i32, 3), :throws => exception(GizzardException)
+    thrift_method :set_remote_cluster_status, void, field(:cluster, string, 1), field(:status, i32, 2), :throws => exception(GizzardException)
+
+    thrift_method :get_remote_host, struct(Host), field(:hostname, string, 1), field(:port, i32, 2), :throws => exception(GizzardException)
+    thrift_method :list_remote_clusters, list(string), :throws => exception(GizzardException)
+    thrift_method :list_remote_hosts, list(struct(Host)), :throws => exception(GizzardException)
+    thrift_method :list_remote_hosts_in_cluster, list(struct(Host)), field(:cluster, string, 1), :throws => exception(GizzardException)
+  end
+
+
+
+  Job = T.make_struct(:Job,
+    T::Field.new(:priority, T::I32, 1),
+    T::Field.new(:contents, T::STRING, 2)
+  )
+
+  JobException = T.make_exception(:JobException,
+    T::Field.new(:description, T::STRING, 1)
+  )
+
+  class JobInjector < GizzmoService
+    thrift_method :inject_jobs, void, field(:priority, list(struct(Job)), 1), :throws => exception(JobException)
   end
 end

data/lib/gizzard/transformation.rb

@@ -0,0 +1,230 @@
+module Gizzard
+  class Transformation
+    require 'gizzard/transformation_op'
+    require 'gizzard/transformation_scheduler'
+
+    OP_NAMES = {
+      Op::RemoveForwarding => "remove_forwarding",
+      Op::RemoveLink       => "remove_link",
+      Op::DeleteShard      => "delete_shard",
+      Op::CreateShard      => "create_shard",
+      Op::AddLink          => "add_link",
+      Op::SetForwarding    => "set_forwarding",
+      Op::CopyShard        => "copy_shard"
+    }
+
+    OP_INVERSES = {
+      Op::AddLink       => Op::RemoveLink,
+      Op::CreateShard   => Op::DeleteShard,
+      Op::SetForwarding => Op::RemoveForwarding
+    }
+
+    OP_INVERSES.keys.each {|k| v = OP_INVERSES[k]; OP_INVERSES[v] = k }
+
+    OP_PRIORITIES = {
+      Op::CreateShard      => 1,
+      Op::AddLink          => 2,
+      Op::SetForwarding    => 3,
+      Op::RemoveForwarding => 4,
+      Op::RemoveLink       => 5,
+      Op::DeleteShard      => 6,
+      Op::CopyShard        => 7
+    }
+
+    DEFAULT_DEST_WRAPPER = 'WriteOnlyShard'
+
+    attr_reader :from, :to
+
+    def initialize(from_template, to_template, copy_dest_wrapper = nil)
+      copy_dest_wrapper ||= DEFAULT_DEST_WRAPPER
+
+      unless Shard::VIRTUAL_SHARD_TYPES.include? copy_dest_wrapper
+        raise ArgumentError, "#{copy_dest_wrapper} is not a valid virtual shard type."
+      end
+
+      @from = from_template
+      @to   = to_template
+      @copy_dest_wrapper = copy_dest_wrapper
+
+      if copies_required? && copy_source.nil?
+        raise ArgumentError, "copy required without a valid copy source"
+      end
+    end
+
+    def bind(base_name, forwardings_to_shards)
+      raise ArgumentError unless forwardings_to_shards.is_a? Hash
+
+      forwardings_to_shards.map do |forwarding, shard|
+        BoundTransformation.new(self, base_name, forwarding, shard)
+      end
+    end
+
+    def inspect
+      op_inspect = operations.inject({}) do |h, (phase, ops)|
+        h.update phase => ops.map {|job| "    #{job.inspect}" }.join("\n")
+      end
+
+      prepare_inspect = op_inspect[:prepare].empty? ? "" : "  PREPARE\n#{op_inspect[:prepare]}\n"
+      copy_inspect    = op_inspect[:copy].empty?    ? "" : "  COPY\n#{op_inspect[:copy]}\n"
+      cleanup_inspect = op_inspect[:cleanup].empty? ? "" : "  CLEANUP\n#{op_inspect[:cleanup]}\n"
+
+      op_inspect = [prepare_inspect, copy_inspect, cleanup_inspect].join
+
+      "#{from.inspect} => #{to.inspect} :\n#{op_inspect}"
+    end
+
+    def operations
+      return @operations if @operations
+
+      log = []
+      log.concat destroy_tree(from) if from
+      log.concat create_tree(to) if to
+
+      # compact
+      log = collapse_jobs(log)
+
+      @operations = expand_jobs(log)
+
+      @operations.each do |(phase, jobs)|
+        jobs.sort!
+      end
+
+      @operations
+    end
+
+    def collapse_jobs(jobs)
+      jobs.reject do |job1|
+        jobs.find do |job2|
+          job1.inverse? job2
+        end
+      end
+    end
+
+    def expand_jobs(jobs)
+      expanded = jobs.inject({:prepare => [], :copy => [], :cleanup => []}) do |ops, job|
+        job_ops = job.expand(self.copy_source, involved_in_copy?(job.template), @copy_dest_wrapper)
+        ops.update(job_ops) {|k,a,b| a + b }
+      end
+
+      # if there are no copies that need to take place, we can do all
+      # nameserver changes in one step
+      if expanded[:copy].empty?
+        expanded[:prepare].concat expanded[:cleanup]
+        expanded[:cleanup] = []
+      end
+
+      expanded
+    end
+
+    def copies_required?
+      return @copies_required unless  @copies_required.nil?
+      @copies_required = !from.nil? &&
+        to.concrete_descendants.reject {|d| from.shared_host? d }.length > 0
+    end
+
+    def involved_in_copy?(template)
+      copy_source?(template) || copy_destination?(template)
+    end
+
+    def copy_destination?(template)
+      copies_required? && template.concrete? && !from.shared_host?(template)
+    end
+
+    def copy_source?(template)
+      copies_required? && !!from.copy_sources.find {|s| s.shard_eql? template }
+    end
+
+    def copy_source
+      from.copy_sources.first if copies_required?
+    end
+
+    def create_tree(root)
+      jobs = visit_collect(root) do |parent, child|
+        [Op::CreateShard.new(child), Op::AddLink.new(parent, child)]
+      end
+      [Op::CreateShard.new(root)].concat jobs << Op::SetForwarding.new(root)
+    end
+
+    def destroy_tree(root)
+      jobs = visit_collect(root) do |parent, child|
+        [Op::RemoveLink.new(parent, child), Op::DeleteShard.new(child)]
+      end
+      [Op::RemoveForwarding.new(root)].concat jobs << Op::DeleteShard.new(root)
+    end
+
+    private
+
+    def visit_collect(parent, &block)
+      parent.children.inject([]) do |acc, child|
+        visit_collect(child, &block).concat(acc.concat(block.call(parent, child)))
+      end
+    end
+  end
+
+
+  class BoundTransformation
+    attr_reader :transformation, :base_name, :forwarding, :shard
+
+    def from; transformation.from end
+    def to;   transformation.to   end
+
+    def initialize(transformation, base_name, forwarding, shard)
+      @transformation = transformation
+      @base_name      = base_name
+      @forwarding     = forwarding
+      @shard          = shard
+
+      @table_id       = forwarding.table_id
+      @base_id        = forwarding.base_id
+      @enum           = shard.enumeration
+      @table_prefix   = Shard.canonical_table_prefix(@enum, @table_id, base_name)
+      @translations   = shard.canonical_shard_id_map(base_name, @table_id, @enum)
+    end
+
+    def prepare!(nameserver)
+      apply_ops(nameserver, transformation.operations[:prepare])
+    end
+
+    def copy_required?
+      !transformation.operations[:copy].empty?
+    end
+
+    def copy!(nameserver)
+      apply_ops(nameserver, transformation.operations[:copy])
+    end
+
+    def cleanup!(nameserver)
+      apply_ops(nameserver, transformation.operations[:cleanup])
+    end
+
+    def involved_shards(phase = :copy)
+      transformation.operations[phase].map do |op|
+        op.involved_shards(@table_prefix, @translations)
+      end.flatten.compact.uniq
+    end
+
+    def involved_hosts(phase = :copy)
+      involved_shards(phase).map {|s| s.hostname }.uniq
+    end
+
+    def inspect
+      "#{@forwarding.inspect}: #{from.inspect} => #{to.inspect}"
+    end
+
+    def copy_descs
+      transformation.operations[:copy].map do |copy|
+        from_id = copy.from.to_shard_id(@table_prefix, @translations)
+        to_id   = copy.to.to_shard_id(@table_prefix, @translations)
+        "#{from_id.inspect} -> #{to_id.inspect}"
+      end
+    end
+
+    private
+
+    def apply_ops(nameserver, ops)
+      ops.each do |op|
+        op.apply(nameserver, @table_id, @base_id, @table_prefix, @translations)
+      end
+    end
+  end
+end