flare-tools 0.1.4 → 0.4.5.1

data/lib/flare/tools/client.rb

@@ -0,0 +1,267 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'timeout'
+require 'flare/net/connection'
+require 'flare/util/logging'
+require 'flare/util/constant'
+require 'flare/util/result'
+
+# 
+module Flare
+  module Tools
+
+    # == Description
+    # 
+    class Client
+      include Flare::Util::Logging
+      extend Flare::Util::Logging
+      include Flare::Util::Constant
+      include Flare::Util::Result
+
+      def self.open(host, port, tout = DefaultTimeout, uplink_limit = DefalutBwlimit, downlink_limit = DefalutBwlimit, &block)
+        session = nil
+        session = self.new(host, port, tout, uplink_limit, downlink_limit)
+        return session if block.nil?
+        return block.call(session)
+      ensure
+        if session.nil?
+          error "failed to open #{host}:#{port}."
+        else
+          session.close # this might raise IOError
+        end
+      end
+
+      def initialize(host, port, tout = DefaultTimeout, uplink_limit = DefalutBwlimit, downlink_limit = DefalutBwlimit)
+        @tout = tout
+        @conn = nil
+        timeout(1) do
+          @conn = Flare::Net::Connection.new(host, port, uplink_limit, downlink_limit)
+        end
+        @server_name, @version = server_version
+      rescue Errno::ECONNREFUSED
+        debug "Connection refused. server=[#{@conn}]"
+        raise
+      rescue TimeoutError
+        debug "Connection timeout. server=[#{@conn}]"
+        raise
+      rescue SocketError
+        debug "Connection error. server=[#{host}:#{port}]"
+        raise
+      end
+
+      def required_version?(required_version, version = @version)
+        (0...required_version.size).each do |i|
+          n = if i < version.size then version[i] else 0 end
+          return true if n > required_version[i]
+          return false if n < required_version[i]
+        end
+        true
+      end
+
+      def host
+        @conn.host
+      end
+
+      def hostname
+        @conn.host
+      end
+
+      def port
+        @conn.port
+      end
+
+      def request(cmd, parser, processor, tout = @tout)
+        # info "request(#{cmd}, #{noreply})"
+        @conn.reconnect if @conn.closed?
+        debug "Enter the command server. server=[#{@conn}] command=[#{cmd}}]"
+        response = nil
+        cmd.chomp!
+        cmd += "\r\n"
+        timeout(tout) do
+          @conn.send(cmd)
+          response = parser.call(@conn, processor)
+        end
+        response
+      rescue TimeoutError => e
+        error "Connection timeout. server=[#{@conn}] command=[#{cmd}}]"
+        @conn.close
+        raise e
+      end
+
+      def sent_size
+        @conn.sent_size
+      end
+
+      def received_size
+        @conn.received_size
+      end
+
+      def close()
+        begin
+          timeout(1) { quit }
+        rescue Timeout::Error => e
+        end
+        @conn.close
+      end
+
+      @@processors = {}
+      @@parsers = {}
+
+      def self.defcmd_generic(method_symbol, command_template, parser, timeout, &default_processor)
+        @@parsers[method_symbol] = parser
+        @@processors[method_symbol] = default_processor || proc { false }
+        timeout_expr = if timeout then "@tout" else "nil" end
+        self.class_eval %{
+          def #{method_symbol.to_s}(*args, &processor)
+            cmd = "#{command_template}"
+            cmd = cmd % args if args.size > 0
+            processor = @@processors[:#{method_symbol}] if processor.nil?
+            request(cmd, @@parsers[:#{method_symbol}], processor, #{timeout_expr})
+          end
+        }
+      end
+
+      def self.defcmd(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          resp = ""
+          answers = [Ok, End, Stored, Deleted, NotFound].map {|x| Flare::Util::Result.string_of_result(x)}
+          fails = [Exists].map {|x| Flare::Util::Result.string_of_result(x)}
+          errors = [Error, ServerError, ClientError].map {|x| Flare::Util::Result.string_of_result(x)}
+          while x = conn.getline
+            ans = x.chomp.split(' ', 2)
+            ans = if ans.empty? then '' else ans[0] end
+            case ans
+            when *answers
+              break
+            when *fails
+              resp = false
+              break
+            when *errors
+              warn "Failed command. server=[#{self}] sent=[#{conn.last_sent}] result=[#{x.chomp}]"
+              resp = false
+              break
+            else
+              resp += x
+            end
+          end
+          return processor.call(resp) if resp 
+          return false
+        }
+        defcmd_generic(method_symbol, command_template, parser, true, &default_processor)
+      end
+
+      def self.defcmd_noreply(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          return processor.call()
+        }
+        defcmd_generic(method_symbol, command_template, parser, true, &default_processor)
+      end
+ 
+      def self.defcmd_oneline(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          line = conn.getline
+          processor.call(line)
+        }
+        defcmd_generic(method_symbol, command_template, parser, true, &default_processor)
+      end
+
+      def self.defcmd_key(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          rets = []
+          while true
+            line = conn.getline
+            elems = line.split(' ')
+            if elems[0] == "KEY"
+              unless processor.nil?
+                r = processor.call(elems[1])
+                rets << r if r
+              end
+            elsif elems[0] == "END"
+              return rets
+            else
+              info "key parser: error \"#{line.chomp}\""
+              return false
+            end
+          end
+        }
+        defcmd_generic(method_symbol, command_template, parser, false, &default_processor)
+      end
+
+      def self.defcmd_value(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          rets = []
+          while true
+            line = conn.getline
+            elems = line.split(' ')
+            if elems[0] == "VALUE"
+              key, flag, len, version, expire = elems[1], elems[2].to_i, elems[3].to_i, elems[4].to_i, elems[5].to_i
+              data = conn.read(len)
+              unless processor.nil?
+                r = processor.call(data, key, flag, len, version, expire)
+                rets << r if r
+              end
+              conn.getline # skip
+            elsif elems[0] == "END"
+              return rets
+            else
+              info "value parser: error \"#{line.chomp}\""
+              return false
+            end
+          end
+        }
+        defcmd_generic(method_symbol, command_template, parser, false, &default_processor)
+      end
+
+      def self.defcmd_listelement(method_symbol, command_template, &default_processor)
+        parser = lambda {|conn,processor|
+          rets = []
+          while true
+            line = conn.getline
+            elems = line.split(' ')
+            if elems[0] == "LISTELEMENT"
+              key, rel, abs = elems[1], elems[2].to_i, elems[3].to_i
+              flag, len, version, expire = elems[4].to_i, elems[5].to_i, elems[6], elems[7]
+              data = conn.read(len)
+              unless processor.nil?
+                r = processor.call(data, key, rel, abs, flag, len, version, expire)
+                rets << r if r
+              end
+              conn.getline # skip
+            elsif elems[0] == "END"
+              return rets[0] if rets.size == 1
+              return rets
+            else
+              info "error \"#{line.chomp}\""
+              return false
+            end
+          end
+        }
+        defcmd_generic(method_symbol, command_template, parser, false, &default_processor)
+      end
+
+      def server_version
+        verstrings = version.split('-')
+        server = "flare"
+        server = verstrings.shift if verstrings.size > 1
+        version = verstrings[0].split('.').map {|v| v.to_i}
+        [server, version]
+      end
+
+      # we have two types of VERSION formats.
+      # VERSION flare-1.0.14
+      # VERSION 1.0.9
+      def version
+        version_
+      end
+      defcmd_oneline :version_, 'version\r\n' do |resp|
+        code, version = resp.chomp.split(' ')
+        return "0.0.0" if code != "VERSION"
+        version          
+      end
+
+    end
+  end
+end

data/lib/flare/tools/cluster.rb

@@ -0,0 +1,319 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'rexml/document'
+require 'flare/util/constant'
+require 'flare/tools/common'
+
+# 
+module Flare
+  module Tools
+
+    # == Description
+    # Cluster is a class that discribes a cluster information.
+    class Cluster
+      include Flare::Util::Constant
+
+      State = 'state'
+      Role  = 'role'
+      StateActive  = 'active'
+      StateDown    = 'down'
+      StateReady   = 'ready'
+      StatePrepare = 'prepare'
+      RoleProxy  = 'proxy'
+      RoleMaster = 'master'
+      RoleSlave  = 'slave'
+      StatPartition = 'partition'
+
+      States = { "active" => '0', "prepare" => '1', "down" => '2', "ready" => '3' }
+      Roles = { "master" => '0', "slave" => '1', "proxy" => '2' }
+
+      class NodeStat
+        def initialize stat
+          @stat = stat.dup
+        end
+
+        def [](i)
+          @stat[i]
+        end
+
+        def []=(i, v)
+          @stat[i] = v.to_s
+        end
+
+        def master?
+          (role == RoleMaster)
+        end
+
+        def slave?
+          (role == RoleSlave)
+        end
+
+        def proxy?
+          (role == RoleProxy)
+        end
+
+        def active?
+          (state == StateActive)
+        end
+
+        def ready?
+          (state == StateReady)
+        end
+        
+        def down?
+          (state == StateDown)
+        end
+
+        def prepare?
+          (state == StatePrepare)
+        end
+
+        def partition
+          @stat['partition'].to_i
+        end
+
+        def thread_type
+          @stat['thread_type'].to_i
+        end
+
+        def balance
+          @stat['balance'].to_i
+        end
+        
+        def method_missing(action, *args)
+          if @stat.has_key? action.to_s
+            @stat[action.to_s]
+          else
+            @stat.__send__(action, *args)
+          end
+        end
+      end
+
+      def initialize(index_server_hostname, index_server_port, nodes_stat)
+        @index_server_hostname = index_server_hostname
+        @index_server_port = index_server_port
+        @nodes_stat = nodes_stat
+        max_partition = -1
+        nodes_stat.each do |nodekey,node_stat|
+          p = node_stat[StatPartition].to_i
+          max_partition = p if p > max_partition
+        end
+        @partition = if max_partition >= 0
+                       (0..max_partition).map {Hash.new}
+                     else
+                       []
+                     end
+        @partition_size = max_partition+1
+        nodes_stat.each do |nodekey,node_stat|
+          p = node_stat[StatPartition].to_i
+          @partition[p][nodekey] = node_stat if p >= 0
+        end
+        @nodes = {}
+        nodes_stat.each do |k,v|
+          @nodes[k] = NodeStat.new(v)
+        end
+      end
+
+      # check if the partition of a nodekey has at least one active slave
+      def reconstructable?(nodekey)
+        node = node_stat(nodekey)
+        ret = if node[State] == StateActive
+                case node[Role]
+                when RoleProxy
+                  false
+                when RoleSlave
+                  true
+                when RoleMaster
+                  # if the partition has at least one active slave, one of the slaves will take over the master.
+                  slaves_in_partition(node[StatPartition]).inject(false) do |r,slave_nodekey|
+                    node_stat(slave_nodekey)[State] == StateActive
+                  end
+                else
+                  error "unknown role: #{node[Role]}"
+                  false
+                end
+              else
+                false
+              end
+        ret
+      end
+
+      def safely_reconstructable?(nodekey)
+        node = node_stat(nodekey)
+        return false if node[State] != StateActive
+        case node[Role]
+        when RoleProxy
+          false
+        when RoleSlave
+          slaves_in_partition(node[StatPartition]).inject(false) do |r, slave_nodekey|
+            if slave_nodekey != nodekey
+              node_stat(slave_nodekey)[State] == StateActive
+            else
+              r
+            end
+          end
+        when RoleMaster
+          count = slaves_in_partition(node[StatPartition]).inject(0) do |r, slave_nodekey|
+            if node_stat(slave_nodekey)[State] == StateActive then r+1 else r end
+          end
+          (count >= 2)
+        else
+          raise "internal error."
+        end
+      end
+
+      def partition(p)
+        @partition[p.to_i]
+      end
+
+      def master_in_partition(p)
+        return nil if partition(p).nil?
+        partition(p).inject(nil) {|r,i|
+          nodekey, node = i
+          if node[Role] == RoleMaster then nodekey else r end
+        }
+      end
+
+      def slaves_in_partition(p)
+        return nil if partition(p).nil?
+        partition(p).inject([]) {|r,i| if i[1][Role] == RoleSlave then r << i[0] else r end}
+      end
+      
+      def nodekeys_(&block)
+        unordered = if block.nil?
+                      @nodes.keys
+                    else
+                      ret = []
+                      @nodes.each do |k,v|
+                        ret << k if block.call(v)
+                      end
+                      ret
+                    end
+        unordered.sort_by do |i|
+          p = @nodes[i].partition
+          p = @partition_size if p < 0
+          [p, @nodes[i].role, i]
+        end
+      end
+      
+      def nodekeys
+        nodekeys_
+      end
+
+      def master_nodekeys
+        nodekeys_ {|v| v[Role] == RoleMaster }
+      end
+
+      def master_and_slave_nodekeys
+        nodekeys_ {|v| v[Role] == RoleMaster || v[Role] == RoleSlave }
+      end
+
+      def node_stat(nodekey)
+        @nodes[nodekey]
+      end
+      
+      def size
+        @nodes.size
+      end
+
+      def partition_size
+        @partition_size
+      end
+
+      # proxy -> -1
+      # not found -> nil
+      def partition_of_nodename node
+        @nodes.each do |k,v|
+          return v[StatPartition].to_i if k == node
+        end
+        return nil
+      end
+
+      def has_nodekey?(nodekey)
+        @nodes.has_key? nodekey
+      end
+
+      def serattr_ x
+        return "" if x.nil?
+        " class_id=\"#{x['class_id']}\" tracking_level=\"#{x['tracking_level']}\" version=\"#{x['version']}\""
+      end
+
+      def serialize
+        thread_type = 0
+
+        node_map_id = {"class_id"=>"0", "tracking_level"=>"0", "version"=>"0"}
+        item_id = {"class_id"=>"1", "tracking_level"=>"0", "version"=>"0"}
+        second_id = {"class_id"=>"2", "tracking_level"=>"0", "version"=>"0"}
+
+        output =<<"EOS"
+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+<!DOCTYPE boost_serialization>
+<boost_serialization signature="serialization::archive" version="4">
+<node_map#{serattr_(node_map_id)}>
+\t<count>#{@nodes.size}</count>
+\t<item_version>0</item_version>
+EOS
+        @nodes.each do |k,v|
+          node_server_name, node_server_port = k.split(':')
+          node_role = Roles[v['role']]
+          node_state = States[v['state']]
+          node_partition = v['partition']
+          node_balance = v['balance']
+          node_thread_type = v['thread_type'].to_i
+          
+          output +=<<"EOS"
+\t<item#{serattr_(item_id)}>
+\t\t<first>#{k}</first>
+\t\t<second#{serattr_(second_id)}>
+\t\t\t<node_server_name>#{node_server_name}</node_server_name>
+\t\t\t<node_server_port>#{node_server_port}</node_server_port>
+\t\t\t<node_role>#{node_role}</node_role>
+\t\t\t<node_state>#{node_state}</node_state>
+\t\t\t<node_partition>#{node_partition}</node_partition>
+\t\t\t<node_balance>#{node_balance}</node_balance>
+\t\t\t<node_thread_type>#{node_thread_type}</node_thread_type>
+\t\t</second>
+\t</item>
+EOS
+          item_id = nil
+          second_id  = nil
+          thread_type = node_thread_type+1 if node_thread_type >= thread_type
+        end
+        output +=<<"EOS"
+</node_map>
+<thread_type>#{thread_type}</thread_type>
+</boost_serialization>
+EOS
+        output
+      end
+
+      def self.build flare_xml
+        doc = REXML::Document.new flare_xml
+        nodemap = doc.elements['/boost_serialization/node_map']
+        thread_type = doc.elements['/boost_serialization/thread_type']
+        count = nodemap.elements['count'].get_text.to_s.to_i
+        item_version = nodemap.elements['item_version'].get_text.to_s.to_i
+        nodestat = []
+        nodemap.elements.each('item') do |item|
+          nodekey = item.elements['first'].get_text.to_s
+          elem = item.elements['second'].elements
+          node = {
+            'server_name' => elem['node_server_name'].get_text.to_s,
+            'server_port' => elem['node_server_port'].get_text.to_s,
+            'role'        => elem['node_role'].get_text.to_s,
+            'state'       => elem['node_state'].get_text.to_s,
+            'partition'   => elem['node_partition'].get_text.to_s,
+            'balance'     => elem['node_balance'].get_text.to_s,
+            'thread_type' => elem['node_thread_type'].get_text.to_s
+          }
+          nodestat << [nodekey, node]
+        end
+        Cluster.new(DefaultIndexServerName, DefaultIndexServerPort, nodestat)
+      end
+
+    end
+  end
+end