flare-tools 0.1.4 → 0.4.5.1

data/bin/flare-admin

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_admin.rb'

data/bin/flare-argv0

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_argv0.rb'

data/bin/flare-deploy

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_argv0.rb'

data/bin/flare-keychecker

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_keychecker.rb'

data/bin/flare-part

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_argv0.rb'

data/bin/flare-ping

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_argv0.rb'

data/bin/flare-stats

@@ -1,12 +1,6 @@
-#! /usr/bin/env ruby
+#!/usr/bin/ruby
 # -*- coding: utf-8; -*-
-# Author::    kgws  (http://d.hatena.ne.jp/kgws/)
-# Copyright:: Copyright (c) 2010- kgws.
-# License::   This program is licenced under the same licence as kgws.
-#
-# $--- flare-partition-setting - [ by Ruby ] $
-# vim: foldmethod=marker tabstop=2 shiftwidth=2
-$: << File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
-require 'flare/tools/cli/stats'
 
-FlareTools::Stats.new.execute
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_argv0.rb'

data/bin/flare-zkadmin

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# -*- coding: utf-8; -*-
+
+$LOAD_PATH.unshift File.dirname(__FILE__)+"/../lib"
+
+require 'flare/tools/cli/flare_zkadmin.rb'

data/lib/flare/net/connection.rb

@@ -0,0 +1,98 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'timeout'
+require 'socket'
+require 'flare/util'
+require 'flare/util/logging'
+require 'flare/util/result'
+require 'flare/util/bwlimit'
+
+# 
+module Flare
+  module Net
+
+    # == Description
+    # 
+    class Connection
+      include Flare::Util::Logging
+      include Flare::Util::Result
+
+      def initialize(host, port, uplink_limit = nil, downlink_limit = nil)
+        @host = host
+        @port = port
+        @socket = TCPSocket.open(host, port)
+        @last_sent = ""
+        @sent_size = 0
+        @received_size = 0
+        @uplink_limit = Flare::Util::Bwlimit.new(uplink_limit)
+        @downlink_limit = Flare::Util::Bwlimit.new(downlink_limit)
+      end
+
+      attr_reader :host, :port
+      attr_reader :sent_size, :received_size
+
+      def close
+        @socket.close unless @socket.closed?
+      end
+
+      def closed?
+        @socket.closed?
+      end
+
+      def reconnect
+        if @socket.closed?
+          @socket = nil
+          @socket = TCPSocket.open(@host, @port)
+        end
+        @socket
+      end
+
+      def send(cmd)
+        if $DEBUG
+          puts "--> server=[#{self}] cmd=[#{cmd.chomp}]"
+        end
+        size = cmd.size
+        @sent_size += size
+        @socket.write cmd
+        @last_sent = cmd
+        @uplink_limit.inc size
+        @uplink_limit.wait
+      end
+
+      def last_sent
+        @last_sent
+      end
+
+      def getline
+        ret = @socket.gets
+        return nil if ret.nil?
+        if $DEBUG
+          puts "<-- #{ret.chomp}"
+        end
+        size = ret.size
+        @received_size += size
+        @downlink_limit.inc size
+        @downlink_limit.wait
+        ret
+      end
+
+      def read(length = nil)
+        ret = @socket.read(length)
+        return nil if ret.nil?
+        size = ret.size
+        @received_size += size
+        @downlink_limit.inc size
+        @downlink_limit.wait
+        ret
+      end
+
+      def to_s
+        "#{@host}:#{@port}"
+      end
+
+    end
+  end
+end

data/lib/flare/test/cluster.rb

@@ -0,0 +1,140 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'uri'
+require 'flare/tools'
+require 'flare/test/daemon'
+require 'flare/test/node'
+
+# 
+module Flare
+  module Test
+
+    # == Description
+    #
+    class Cluster
+      include Flare::Tools::Common
+
+      def initialize(name, option = {})
+        if ENV.has_key?("FLARE_INDEX_DB") && !option.has_key?("index-db")
+          option["index-db"] = ENV["FLARE_INDEX_DB"]
+        end
+
+        if option.has_key?("index-db")
+          uri = URI.parse(option["index-db"])
+          if uri.scheme == "zookeeper"
+            z = ::Zookeeper.new("#{uri.host}:#{uri.port}")
+            Flare::Tools::ZkUtil.clear_nodemap z, uri.path
+            z.close
+          end
+        end
+
+        daemon = Daemon.instance
+        @indexport = daemon.assign_port
+        @workdir = Dir.pwd+"/work"
+        @datadir = [@workdir, "#{name}.#{@indexport}"].join('/')
+        @nodes = {}
+
+        @indexname = "localhost"
+        @index_pid = daemon.invoke_flarei(name, {
+                                            'server-name' => @indexname,
+                                            'server-port' => @indexport,
+                                            'data-dir' => @datadir,
+                                          }.merge(option))
+        @flare_xml = [@datadir, "flare.xml"].join('/')
+        sleep 1
+      end
+
+      def indexname
+        @indexname
+      end
+
+      def indexport
+        @indexport
+      end
+      
+      def create_node(name, config = {}, executable = Daemon::Flared)
+        daemon = Daemon.instance
+        serverport = daemon.assign_port
+
+        datadir = [@datadir, "#{name}.#{serverport}"].join('.')
+
+        servername = "localhost"
+        pid = daemon.invoke_flared(name, {
+                                     'index-server-name' => @indexname,
+                                     'index-server-port' => @indexport,
+                                     'server-name' => servername,
+                                     'server-port' => serverport,
+                                     'data-dir' => datadir,
+                                   }.merge(config), executable)
+        hostname_port = "#{servername}:#{serverport}"
+        node = @nodes[hostname_port] = Node.new(hostname_port, pid)
+        node
+      end
+      
+      def shutdown
+        daemon = Daemon.instance
+        daemon.shutdown
+      end
+
+      def nodes
+        @nodes.values
+      end
+
+      def wait_for_ready
+        Flare::Tools::IndexServer.open(indexname, indexport, 10) do |s|
+          wait_for_servers(s)
+        end        
+      end
+
+      def prepare_master_and_slaves(nodes, partition = 0)
+        Flare::Tools::IndexServer.open(indexname, indexport, 10) do |s|
+          slaves = nodes.dup
+          master = slaves.shift
+          
+          # master
+          s.set_role(master.hostname, master.port, 'master', 1, partition)
+          wait_for_master_construction(s, "#{master.hostname}:#{master.port}", 10)
+          s.set_state(master.hostname, master.port, 'active')
+          
+          # slave
+          slaves.each do |n|
+            s.set_role(n.hostname, n.port, 'slave', 1, partition)
+            wait_for_slave_construction(s, "#{n.hostname}:#{n.port}", 10, true)
+          end
+        end
+      end
+
+      def prepare_data(node, prefix, count)
+        Flare::Tools::Node.open(node.hostname, node.port, 10) do |n|
+          fmt = "#{prefix}%010.10d"
+          (0...count).each do |i|
+            n.set(fmt % i, "All your base are belong to us.")
+          end
+        end
+      end
+
+      def clear_data(node)
+        Flare::Tools::Node.open(node.hostname, node.port, 10) do |n|
+          n.flush_all
+        end
+      end
+
+      def exist?(node)
+        return Flare::Tools::IndexServer.open(indexname, indexport, 10) do |s|
+          nodes_stats = s.stats_nodes
+          nodes_stats.has_key? node
+        end
+      end
+
+      def index
+        open(@flare_xml) do |f|
+          f.read
+        end
+      end
+
+    end
+  end
+end

data/lib/flare/test/daemon.rb

@@ -0,0 +1,144 @@
+# -*- coding: utf-8; -*-
+# Authors::   Kiyoshi Ikehara <kiyoshi.ikehara@gree.net>
+# Copyright:: Copyright (C) GREE, Inc. 2011.
+# License::   MIT-style
+
+require 'timeout'
+require 'singleton'
+require 'thread'
+
+require 'flare/util/flarei_conf'
+require 'flare/util/flared_conf'
+
+# 
+module Flare
+  module Test
+    class Daemon
+      include Singleton
+  
+      Flarei = "/usr/local/bin/flarei"
+      Flared = "/usr/local/bin/flared"
+
+      FlareiVersion = `#{Flarei} -v`.chomp.split(' ')[-1]
+      FlaredVersion = `#{Flared} -v`.chomp.split(' ')[-1]
+
+      def initialize
+        @flared = []
+        @flarei = []
+        @port = 20000+rand(10000)
+        @port_mutex = Mutex.new
+        @tempfiles = []
+        Kernel.at_exit {
+          Flare::Test::Daemon.instance.shutdown
+        }
+      end
+
+      def shutdown
+        STDERR.print "killing..."
+        (@flared+@flarei).each do |pid|
+          STDERR.print " #{pid}"
+          begin
+            timeout(10) do
+              Process.kill :TERM, pid
+              Process.waitpid pid
+            end
+          rescue Errno::ESRCH
+            STDERR.print "?"
+          rescue TimeoutError => e
+            Process.kill :KILL, pid
+            Process.waitpid pid
+            STDERR.print "*"
+          end
+        end
+        STDERR.print "\n"
+        Process.waitall
+        @flared.clear
+        @flarei.clear
+        @tempfiles.each do |datadir|
+          deleteall(datadir)
+        end
+        @tempfiles.clear
+      end
+
+      def required_version? required_version
+        version = FlareiVersion.split('.').map {|i| i.to_i}
+        (0...required_version.size).each do |i|
+          n = if i < version.size then version[i] else 0 end
+          return true if n > required_version[i]
+          return false if n < required_version[i]
+        end
+        true
+      end
+
+      def assign_port
+        port = 0
+        @port_mutex.synchronize do
+          port = @port
+          @port += 1
+        end
+        port
+      end
+
+      def invoke_flarei(name, config, executable = Flarei)
+        config["key-hash-algorithm"] = "crc32" if required_version? [1, 0, 15]
+        config = Flare::Util::FlareiConf.new(config)
+        conf = "/tmp/flarei.#{name}.#{config.server_port}.conf"
+        open(conf, "w") do |f|
+          f.puts config.to_s
+        end
+        pid = fork
+        if pid.nil?
+          deleteall(config.data_dir)
+          Dir.mkdir(config.data_dir)
+          begin
+            exec executable, "-f", conf
+          rescue
+            exit 1
+          end
+        else
+          @flarei << pid
+          @tempfiles << config.data_dir
+          @tempfiles << conf
+        end
+        pid
+      end
+
+      def invoke_flared(name, config, executable = Flared)
+        config = Flare::Util::FlaredConf.new(config)
+        conf = "/tmp/flared.#{name}.#{config.server_port}.conf"
+        open(conf, "w") do |f|
+          f.puts config.to_s
+        end
+        pid = fork
+        if pid.nil?
+          deleteall(config.data_dir)
+          Dir.mkdir(config.data_dir)
+          begin
+            exec executable, "-f", conf
+          rescue
+            exit 1
+          end
+        else
+          @flared << pid
+          @tempfiles << config.data_dir
+          @tempfiles << conf
+        end
+        pid
+      end
+
+      def deleteall(delthem)
+        return unless FileTest.exist?(delthem)
+        if FileTest.directory?(delthem)
+          Dir.foreach(delthem) do |file|
+            next if /^\.+$/ =~ file
+            deleteall(delthem.sub(/\/+$/,"") + "/" + file)
+          end
+          Dir.rmdir(delthem) rescue ""
+        else
+          File.delete(delthem)
+        end
+      end
+
+    end
+  end
+end