gizzmo 0.11.0 → 0.11.1

data/lib/gizzard/migrator.rb

@@ -0,0 +1,192 @@
+module Gizzard
+  class MigratorConfig
+    attr_accessor :prefix, :table_id, :source_type, :destination_type, :forwarding_space, :forwarding_space_min, :manifest
+
+    def initialize(opts = {})
+      opts.each {|(k,v)| send("#{k}=", v) if respond_to? "{k}=" }
+    end
+
+    def shard_name(enum)
+      table_id_segment = (table_id && table_id < 0) ? "n#{table_id.abs}" : table_id
+      [prefix, table_id, "%04d" % enum].compact.join("_")
+    end
+  end
+
+  class Migrator
+    BALANCE_TOLERANCE = 1
+
+    attr_reader :configured_templates, :existing_map, :existing_templates, :total_shards
+
+    # populated via derive_changes
+    attr_reader :new_templates, :unrecognized_templates, :similar_templates, :unchanged_templates
+
+    def initialize(existing_map, config_templates, default_total_shards, config)
+      @configured_templates = config_templates
+      @existing_map = existing_map
+
+      @existing_templates = existing_map.keys
+      @total_shards = @existing_map.values.map { |a| a.length }.inject { |a, b| a + b } || default_total_shards
+      @config = config
+
+      derive_changes
+    end
+
+    def prepare!(nameserver)
+      transformations.each {|t| t.prepare! nameserver, @config }
+    end
+
+    def copy!(nameserver)
+      transformations.each {|t| t.copy! nameserver, @config }
+    end
+
+    def wait_for_copies(nameserver)
+      transformations.each {|t| t.wait_for_copies nameserver, @config }
+    end
+
+    def cleanup!(nameserver)
+      transformations.each {|t| t.cleanup! nameserver, @config }
+    end
+
+    def transformations
+      return @transformations if @transformations
+
+      # no changes
+      return @transformations = [] if similar_templates.empty? and unrecognized_templates.empty? and new_templates.empty?
+
+      configured_map = configured_templates.inject({}) {|h, t| h.update t => [] }
+
+      @transformations = []
+
+      if existing_templates.empty?
+        # no forwardings exist, we must populate the forwarding index.
+        forwardings = generate_new_forwardings(total_shards)
+
+        # add the new shard ids to a member of the configured map. will
+        # be rebalanced later.
+        configured_map.values.first.concat forwardings.values
+
+        @transformations << ForwardingTransformation.new(@config.table_id, forwardings.inject({}) {|f, (b, e)| f.update b => @config.shard_name(e) })
+      end
+
+      # map the unchanged templates straight over
+      move_unchanged(existing_map, configured_map)
+
+      # map similar templates over to their new versions
+      move_similar(existing_map, configured_map)
+
+      # move shards from unrecognized templates to new templates (or
+      # existing ones)
+      move_unrecognized_to_new(existing_map, configured_map)
+
+      # rebalance
+      rebalance_shards(configured_map)
+
+      # transformation generation
+      @transformations = generate_transformations(existing_map, configured_map) + @transformations
+    end
+
+    private
+
+    def generate_new_forwardings(shard_count)
+      forwardings = {}
+      step_size = @config.forwarding_space / shard_count
+      bases = (0...shard_count).map { |i| @config.forwarding_space_min + (i * step_size) }
+
+      bases.each_with_index do |base_id, i|
+        forwardings[base_id] = i
+      end
+
+      forwardings
+    end
+
+    def move_unchanged(existing, configured)
+      unchanged_templates.each {|u| configured[u] = existing[u].dup }
+    end
+
+    def move_similar(existing, configured)
+      similar_templates.each {|from, to| configured[to] = existing[from].dup }
+    end
+
+    def move_unrecognized_to_new(existing, configured)
+      # duplicate so we can mutate our copy
+      unrecognized = unrecognized_templates.dup
+
+      # for each new template, grab an unrecognized one's shards
+      # and pop it off
+      new_templates.each do |n|
+        if u = unrecognized.pop
+          configured[n] = existing[u].dup
+        end
+      end
+
+      # if there are any unrecognized templates for which we haven't
+      # moved shards over, add their shards to the first template. they will get rebalanced later
+      leftover_shards = unrecognized.inject([]) {|a, u| a.concat existing[u] }
+
+      configured.values.last.concat leftover_shards unless leftover_shards.empty?
+    end
+
+    def rebalance_shards(configured)
+      until shards_balanced? configured
+        smallest(configured) << largest(configured).pop
+      end
+    end
+
+    def generate_transformations(existing, configured)
+      existing_shards = shards_to_templates(existing)
+      configured_shards = shards_to_templates(configured)
+
+      # find the list of shards which have moved, and emit a
+      # transformation for each one.
+      (configured_shards.to_a - existing_shards.to_a).inject({}) do |transformations, (shard, to)|
+        from = existing_shards[shard]
+        (transformations[[from, to]] ||= Transformation.new(from, to, [])).shards << shard
+        transformations
+      end.values
+    end
+
+    def shards_balanced?(template_map)
+      sorted_sizes = template_map.values.map {|s| s.length }.uniq.sort.reverse
+      sorted_sizes.first - sorted_sizes.last <= BALANCE_TOLERANCE
+    end
+
+    def smallest(template_map)
+      template_map.values.sort {|a,b| a.length <=> b.length }.first
+    end
+
+    def largest(template_map)
+      template_map.values.sort {|a,b| b.length <=> a.length }.first
+    end
+
+    def shards_to_templates(templates_to_shards)
+      templates_to_shards.inject({}) do |h, (template, shards)|
+        shards.each {|shard| h[shard] = template }; h
+      end
+    end
+
+    def derive_changes
+      @unrecognized_templates, @new_templates, related_templates =
+        split_set(existing_templates, configured_templates) {|a, b| a.similar? b }
+
+      @similar_templates = related_templates.reject {|(a,b)| a == b }
+      @unchanged_templates = related_templates.keys - @similar_templates.keys
+    end
+
+    def split_set(a, b, &predicate)
+      in_a = a.dup
+      in_b = b.dup
+      overlap = {}
+
+      in_a.each_with_index do |a, a_i|
+        in_b.each_with_index do |b, b_i|
+          if predicate.call(a, b)
+            overlap[a] = b
+            in_a[a_i] = in_b[b_i] = nil
+          end
+        end
+      end
+
+      [in_a.compact, in_b.compact, overlap]
+    end
+  end
+end

data/lib/gizzard/nameserver.rb

@@ -0,0 +1,206 @@
+module Gizzard
+  Shard = Struct.new(:info, :children, :weight)
+
+  class Shard
+    class << self
+      def canonical_table_prefix(enum, table_id = nil, base_prefix = "shard")
+        enum_s         = "%0.4i" % enum
+        table_id_s     = table_id.nil? ? nil : table_id < 0 ? "n#{table_id.abs}" : table_id.to_s
+        [base_prefix, table_id_s, enum_s].compact.join('_')
+      end
+
+      def parse_enumeration(table_prefix)
+        if match = table_prefix.match(/\d{3,}/)
+          match[0].to_i
+        else
+          raise "Cannot derive enumeration!"
+        end
+      end
+    end
+
+    VIRTUAL_SHARD_TYPES = [
+      "FailingOverShard",
+      "ReplicatingShard",
+      "ReadOnlyShard",
+      "WriteOnlyShard",
+      "BlockedShard",
+    ]
+
+    REPLICATING_SHARD_TYPES = ["ReplicatingShard", "FailingOverShard"]
+
+    INVALID_COPY_TYPES = ["ReadOnlyShard", "WriteOnlyShard", "BlockedShard"]
+
+    SHARD_SUFFIXES = {
+      "FailingOverShard" => 'replicating',
+      "ReplicatingShard" => 'replicating',
+      "ReadOnlyShard" => 'read_only',
+      "WriteOnlyShard" => 'write_only',
+      "BlockedShard" => 'blocked'
+    }
+
+    def id; info.id end
+    def hostname; id.hostname end
+    def table_prefix; id.table_prefix end
+    def class_name; info.class_name end
+    def source_type; info.source_type end
+    def destination_type; info.destination_type end
+    def busy; info.busy end
+
+    def template
+      child_templates = children.map {|c| c.template }
+
+      ShardTemplate.new(info.class_name,
+                        id.hostname,
+                        weight,
+                        info.source_type,
+                        info.destination_type,
+                        child_templates)
+    end
+
+    def enumeration
+      self.class.parse_enumeration(table_prefix)
+    end
+
+    def canonical_shard_id_map(base_prefix = "shard", table_id = nil, enum = nil)
+      enum         ||= self.enumeration
+      base           = Shard.canonical_table_prefix(enum, table_id, base_prefix)
+      suffix         = SHARD_SUFFIXES[class_name.split('.').last]
+      canonical_name = [base, suffix].compact.join('_')
+      canonical_id   = ShardId.new(self.hostname, canonical_name)
+
+      children.inject(canonical_id => self.id) do |m, c|
+        m.update c.canonical_shard_id_map(base_prefix, table_id, enum)
+      end
+    end
+  end
+
+  class Nameserver
+
+    DEFAULT_PORT    = 7917
+    DEFAULT_RETRIES = 20
+    PARALLELISM     = 10
+
+    attr_reader :hosts, :logfile, :dryrun, :framed
+    alias dryrun? dryrun
+
+    def initialize(*hosts)
+      options = hosts.last.is_a?(Hash) ? hosts.pop : {}
+      @retries = options[:retries] || DEFAULT_RETRIES
+      @logfile = options[:log]     || "/tmp/gizzmo.log"
+      @dryrun  = options[:dry_run] || false
+      @framed  = options[:framed]  || false
+      @hosts   = hosts.flatten
+    end
+
+    def get_shards(ids)
+      ids.map {|id| with_retry { client.get_shard(id) } }
+    end
+
+    def reload_config
+      all_clients.each {|c| with_retry { c.reload_config } }
+    end
+
+    def copy_shard(from_shard_id, to_shard_id)
+      c = random_client
+      with_retry { c.copy_shard(from_shard_id, to_shard_id) }
+    end
+
+    def respond_to?(method)
+      client.respond_to? method or super
+    end
+
+    def method_missing(method, *args, &block)
+      if client.respond_to?(method)
+        with_retry { client.send(method, *args, &block) }
+      else
+        super
+      end
+    end
+
+    def manifest(*table_ids)
+      Manifest.new(self, table_ids)
+    end
+
+    private
+
+    def client
+      @client ||= create_client(hosts.first)
+    end
+
+    def all_clients
+      @all_clients ||= hosts.map {|host| create_client(host) }
+    end
+
+    def random_client
+      all_clients[rand(all_clients.length)]
+    end
+
+    def create_client(host)
+      host, port = host.split(":")
+      port ||= DEFAULT_PORT
+      Manager.new(host, port.to_i, logfile, framed, dryrun)
+    end
+
+    private
+
+    def with_retry
+      times ||= @retries
+      yield
+    rescue ThriftClient::Simple::ThriftException, NoMethodError, Gizzard::GizzardException => e
+      raise if e.is_a? Gizzard::GizzardException and e.message !~ /Communications link failure/
+
+      times -= 1
+      (times < 0) ? raise : (sleep 2; retry)
+    end
+
+    class Manifest
+      attr_reader :forwardings, :links, :shard_infos, :trees, :templates
+
+
+      def initialize(nameserver, table_ids)
+        states = table_ids.map {|id| nameserver.dump_nameserver(id) }
+
+        @forwardings = states.map {|s| s.forwardings }.flatten
+
+        @links = states.map {|s| s.links }.flatten.inject({}) do |h, link|
+          (h[link.up_id] ||= []) << [link.down_id, link.weight]; h
+        end
+
+        @shard_infos = states.map {|s| s.shards }.flatten.inject({}) do |h, shard|
+          h.update shard.id => shard
+        end
+
+        @trees = @forwardings.inject({}) do |h, forwarding|
+          h.update forwarding => build_tree(forwarding.shard_id)
+        end
+
+        @templates = @trees.inject({}) do |h, (forwarding, shard)|
+          (h[shard.template] ||= []) << forwarding; h
+        end
+      end
+
+      private
+
+      def get_filtered_forwardings(nameserver, filter)
+        return filter[:forwardings] if filter[:forwardings]
+
+        forwardings = nameserver.get_forwardings
+
+        if table_id = filter[:table_id]
+          forwardings.reject! {|f| f.table_id != table_id }
+        end
+
+        forwardings
+      end
+
+      def build_tree(shard_id, link_weight=ShardTemplate::DEFAULT_WEIGHT)
+        children = (links[shard_id] || []).map do |(child_id, child_weight)|
+          build_tree(child_id, child_weight)
+        end
+
+        info = shard_infos[shard_id] or raise "shard info not found for: #{shard_id}"
+        Shard.new(info, children, link_weight)
+      end
+    end
+  end
+end

data/lib/gizzard/shard_template.rb

@@ -0,0 +1,252 @@
+module Gizzard
+  class ShardTemplate
+    include Comparable
+
+    ABSTRACT_HOST = "localhost"
+    DEFAULT_WEIGHT = 1
+
+    attr_reader :type, :weight, :source_type, :dest_type
+
+    def initialize(type, host, weight, source_type, dest_type, children)
+      @type, @host, @weight, @source_type, @dest_type, @children =
+        type, host, weight, source_type || '', dest_type || '', children
+    end
+
+    def self.concrete?(type)
+      !Shard::VIRTUAL_SHARD_TYPES.include? type.split('.').last
+    end
+
+    def concrete?
+      self.class.concrete? type
+    end
+
+    def replicating?
+      Shard::REPLICATING_SHARD_TYPES.include? type.split('.').last
+    end
+
+    def valid_copy_source?
+      !Shard::INVALID_COPY_TYPES.include? type.split('.').last
+    end
+
+    def identifier
+      concrete? ? "#{type}/#{host}" : type.to_s
+    end
+
+    def table_name_suffix
+      Shard::SHARD_SUFFIXES[type.split('.').last]
+    end
+
+    def host
+      if concrete?
+        @host
+      elsif replicating?
+        ABSTRACT_HOST
+      else
+        children.first.host
+      end
+    end
+
+    def children
+      @children.sort { |a, b| b <=> a }
+    end
+
+    def descendants
+      [self].concat children.map {|c| c.descendants }.flatten
+    end
+
+    alias flatten descendants
+
+    def concrete_descendants
+      descendants.select {|t| t.concrete? }
+    end
+
+    def copy_sources
+      return [] unless self.valid_copy_source?
+      self.concrete? ? [self] : children.inject([]) {|a, c| a.concat c.copy_sources }
+    end
+
+    def inspect
+      to_config
+    end
+    alias to_s inspect
+
+    # Concretization
+
+    def to_shard_id(table_prefix, translations = {})
+      table_prefix = [table_prefix, table_name_suffix].compact.join('_')
+      shard_id     = ShardId.new(host, table_prefix)
+      translations[shard_id] || shard_id
+    end
+
+    def to_shard_info(table_prefix, translations = {})
+      ShardInfo.new(to_shard_id(table_prefix, translations), type, source_type, dest_type, 0)
+    end
+
+    def to_shard(table_prefix, translations = {})
+      Shard.new(to_shard_info(table_prefix, translations), children.map {|c|
+        c.to_shard(table_prefix, translations)
+      }, weight)
+    end
+
+    # Similarity/Equality
+
+    def <=>(other)
+      raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
+
+      to_a = lambda {|t| [t.host, t.type, t.source_type.to_s, t.dest_type.to_s, t.weight] }
+
+      if (cmp = to_a.call(self) <=> to_a.call(other)) == 0
+        children <=> other.children
+      else
+        cmp
+      end
+    end
+
+    def eql?(other)
+      return false unless other.is_a? ShardTemplate
+      (self <=> other) == 0
+    end
+
+    def shard_eql?(other)
+      raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
+
+      to_a = lambda {|t| [t.host, t.type, t.source_type.to_s, t.dest_type.to_s] }
+      to_a.call(self) == to_a.call(other)
+    end
+
+    def link_eql?(other)
+      raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
+
+      to_a = lambda {|t| [t.host, t.type, t.source_type.to_s, t.dest_type.to_s, t.weight] }
+      to_a.call(self) == to_a.call(other)
+    end
+
+    def shared_host?(other)
+      raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
+
+      (self.concrete_descendants & other.concrete_descendants).length > 0
+    end
+
+    def hash
+      weight.hash + host.hash + type.hash + children.hash
+    end
+
+
+    # Config
+
+    def config_definition
+      args = identifier.split("/")
+      args << weight
+      args.concat [@source_type,@dest_type] unless [@source_type, @dest_type].reject {|s| s.empty? }.empty?
+
+      type = args.shift
+      args_s = args.empty? ? "" : "(#{args.join(",")})"
+
+      type + args_s
+    end
+
+    private :config_definition
+
+    def to_config_struct
+      if children.empty?
+        config_definition
+      else
+        child_defs = children.map {|c| c.to_config_struct }
+        { config_definition => (child_defs.length == 1 ? child_defs.first : child_defs) }
+      end
+    end
+
+    def to_config
+      if children.empty?
+        config_definition
+      else
+        child_defs = children.map {|c| c.to_config }
+        child_defs_s = child_defs.length == 1 ? child_defs.first : "(#{child_defs.join(", ")})"
+        "#{config_definition} -> #{child_defs_s}"
+      end
+    end
+
+
+    # Class Methods
+
+    class << self
+      def parse(string)
+        definition_s, children_s = string.split(/\s*->\s*/, 2)
+
+        children =
+        if children_s.nil?
+          []
+        else
+          list = parse_arg_list(children_s).map {|c| parse c }
+          raise ArgumentError, "invalid shard config. -> given, no children found" if list.empty?
+          list
+        end
+
+        template_args = parse_definition(definition_s) << children
+        ShardTemplate.new(*template_args)
+      end
+
+      private
+
+      def parse_definition(definition_s)
+        type, arg_list = definition_s.split("(", 2)
+
+        host, weight, source_type, dest_type =
+          if arg_list.nil?
+            nil
+          else
+            args = parse_arg_list("(" + arg_list)
+            args.unshift nil unless concrete? type
+            args
+          end
+
+        validate_host_arg(host, definition_s) if concrete? type
+        validate_weight_arg(weight, definition_s)
+
+        weight = (weight || DEFAULT_WEIGHT).to_i
+        source_type ||= ""
+        dest_type   ||= ""
+
+        [type, host, weight, source_type, dest_type]
+      end
+
+      def parse_arg_list(string)
+        string = string.strip
+        if m = string.match(/\A\((.*)\)\Z/)
+          string = m[1]
+        end
+
+        depth = 0
+        results = [[]]
+
+        string.each_char do |c|
+          case c
+          when ","
+            if depth == 0
+              results << []
+              next
+            end
+          when "(" then depth += 1
+          when ")" then depth -= 1
+          end
+
+          results.last << c
+        end
+
+        results.map {|r| r.join.strip }
+      end
+
+      def validate_weight_arg(arg, definition)
+        if arg && YAML.load(arg.to_s).is_a?(String)
+          raise ArgumentError, "Invalid weight #{arg} for shard in: #{definition}"
+        end
+      end
+
+      def validate_host_arg(arg, definition)
+        if arg.nil? || YAML.load(arg.to_s).is_a?(Numeric)
+          raise ArgumentError, "Invalid host #{arg} for shard in: #{definition}"
+        end
+      end
+    end
+  end
+end