internethakai 0.2.0

data/lib/internethakai/reporter.rb

@@ -0,0 +1,143 @@
+module InternetHakai
+    class Reporter < BaseHandler
+        UNIQUE_BY_THREAD = false
+        def run record
+            @config = BaseHandler.get_config
+            @logger = BaseHandler::get_handler(@config["logger"])
+        end
+        def set_dir dir
+            @dir = dir
+        end
+        def init_filename fname
+            @filename = fname
+        end
+        def get_filename
+            @filename
+        end
+    end
+    class PStoreReporter < Reporter
+        require 'pstore'
+        def run record
+            super
+            return unless record
+            fpath = File::join(@dir, @filename)
+            @logger.run("save to #{fpath}\n", 3)
+            db = PStore::new(fpath)
+            db.transaction do
+                db["result"] = ResponseRecord::new unless db.root?('result')
+                db['result'] += record
+            end
+        end
+    end
+    #TSV形式で保存するクラス
+    class TsvReporter < Reporter
+        UNIQUE_BY_THREAD = false
+        def init_filename starttime
+            @filename = "report_#{starttime}.tsv"
+        end
+        def run record
+            super
+            if not record
+                @logger.run("no rocord\n", 2)
+                exit -1
+            end
+            keys = record.keys.to_a.sort
+            strResultTSV = "Path\tCount\tAvgResponse(ms)\tMinResponse(ms)\tMaxResponse(ms)\tAvgContentLength\tError%\tHTTP_STATUS_CODE\ttime(ms)\n"
+
+            rank_avg_time = []
+            sum_avg_time = 0
+            rank_error_per = []
+            sum_error = 0
+            sum_access = 0
+
+            if keys.nil?
+                @logger.run("cannot show report\n", 2)
+                exit -1
+            end
+            keys.to_a.each do |key|
+                v = record[key]
+                next if v[:accesscount] == 0
+                avg_res  = v[:totaltime] * 1000.0 / v[:accesscount]
+                error_per = (v[:errorcount].to_f / v[:accesscount])*100
+
+                rank_avg_time << [key, avg_res, v[:accesscount], v[:min]*1000.0]
+                sum_avg_time += v[:totaltime] * 1000.0
+                rank_error_per << [key, error_per, v[:accesscount]]
+                sum_error += v[:errorcount]
+                sum_access += v[:accesscount]
+
+                strResultTSV +=
+                    key + "\t" +                                                      # URL
+                    v[:accesscount].to_s + "\t" +                                     # COUNT
+                    format( '%f', avg_res ) + "\t" +                                  # AVG
+                    format('%f', v[:min].to_f*1000) + "\t" +
+                    format('%f', v[:max].to_f*1000) + "\t" +
+                    format( '%d', v[:size] / v[:accesscount] ) + "\t" +               # AVG
+                    (error_per).to_s + "\t"              # ERROR%
+                status_code = []
+                res_time =[]
+                v.keys.each do |v_key|
+                    if(v_key.to_s =~ /^status:/)
+                        status_code << "#{v_key}: #{v[v_key]}"
+                    end
+                    if(v_key.to_s =~ /^time:/)
+                        res_time << "#{v_key}: #{v[v_key]}"
+                    end
+                end
+                strResultTSV += status_code.sort.join(",") + "\t"                        # HTTP_STAT
+                strResultTSV += res_time.sort.join(",")                                 # RESPONSE TIME
+                strResultTSV += "\n"
+            end
+
+            if @config["save_report"]
+                save strResultTSV
+                fpath = File::join(@dir, @filename)
+                @logger.run("save to:"+fpath+"\n", 2)
+            end
+
+
+            #ranking
+            rank_error_per = rank_error_per.sort_by{|p|p[1]}.reverse.slice(0,@config["ranking"]).to_a
+            rank_avg_time = rank_avg_time.sort_by{|p|p[1]}.reverse.slice(0,@config["ranking"]).to_a
+            rank_min_time = rank_avg_time.sort_by{|p|p[3]}.reverse.slice(0,@config["ranking"]).to_a
+
+            @logger.run("MinResponse Worst:\n", 2)
+            rank = 1
+            rank_min_time.each do |p|
+                @logger.run("\t#{rank}: #{p[0]} #{p[3]} #{p[2]}\n", 2)
+                rank += 1
+            end
+            @logger.run("AvgResponse Worst:\n", 2)
+            rank = 1
+            rank_avg_time.each do |p|
+                @logger.run("\t#{rank}: #{p[0]} #{p[1]} #{p[2]}\n", 2)
+                rank += 1
+            end
+            @logger.run("AvgResponse Total:\n", 2)
+            @logger.run("\t#{sum_avg_time.to_f / sum_access}\n", 2)
+
+            @logger.run("ErrorRate Worst:\n", 2)
+            rank = 1
+            rank_error_per.each do |p|
+                @logger.run("\t#{rank}: #{p[0]} #{p[1]} #{p[2]}\n", 2)
+                rank += 1
+            end
+            @logger.run("ErrorRate Total:\n", 2)
+            @logger.run("\t#{(sum_error.to_f / sum_access)*100.0}\n", 2)
+
+            if(@config["show_report"])
+                @logger.run(strResultTSV, 2)
+            end
+        end
+        def save str
+            begin
+                fpath = File::join(@dir, @filename)
+                File::open(fpath, "w") do |io|
+                    io.print str
+                end
+            rescue
+                raise $!
+            end
+        end
+    end
+end

data/lib/internethakai/response.rb

@@ -0,0 +1,65 @@
+module InternetHakai
+    #http response
+    class ResponseObject
+        def self::create_from_nethttp obj
+            r = self::new
+            r.body = obj.body.to_s
+            r.status = obj.code.to_i
+            r.content_type = obj['content-type']
+            r.location = obj['location'] if obj['location']
+            r.cookie = obj['set-cookie']
+            return r
+        end
+        def initialize
+            @header = nil
+            @status = 0
+            @time = 0
+            @content_type = ''
+            @location = nil
+            @body = ''
+            @cookie = nil
+        end
+        attr_accessor :body, :header, :status, :time, :content_type, :location, :cookie
+    end
+    #レスポンスタイム記録クラス
+    class  ResponseRecord < Hash
+        ## 参照
+        def []( key )
+            if (! self.has_key?( key ))
+                self[key] = {
+                    :totaltime    =>    0,
+                    :accesscount  =>    0,
+                    :errorcount   =>    0,
+                    :size         =>    0,
+                    :min          =>    nil,
+                    :max          =>    nil
+                }
+            end
+            return( super(key) )
+        end
+        # 演算
+        def +(other)
+            other.each do |key, value|
+                s = self[key]
+                s[:totaltime]     += value[:totaltime]
+                s[:accesscount]   += value[:accesscount]
+                s[:errorcount]    += value[:errorcount]
+                s[:size]          += value[:size]
+                if s[:min].nil?
+                    s[:min] = value[:min]
+                    s[:max] = value[:max]
+                else
+                    s[:min] = (s[:min] < value[:min]) ? s[:min] : value[:min] unless value[:min].nil?
+                    s[:max] = (s[:max] > value[:max]) ? s[:max] : value[:max] unless value[:max].nil?
+                end
+
+                value.keys.each do |v_key|
+                    if(v_key.to_s =~ /status:/ || v_key.to_s =~ /time:/)
+                        s[v_key] = s[v_key].to_i + value[v_key]
+                    end
+                end
+            end
+            return self
+        end
+    end
+end

data/lib/internethakai/scenario.rb

@@ -0,0 +1,98 @@
+module InternetHakai
+    # = シナリオ
+    class Scenario
+        def initialize(opt)
+            @config = opt
+            @size = 0
+            @actions_size
+            @performance_id = nil
+            @scenario_id = nil
+            @loop = @config['loop']
+            @loop_org = @loop
+            digit = @loop.to_s.size
+            @tmpl = "%0#{digit}d"
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+
+            #@page_size = [10, @loop].min
+            @page_size = 1
+            @page = (@loop.quo(@page_size)).ceil
+            @page_counter = @page
+            @vars = {}
+            @cookie = {}
+        end
+        attr_reader :actions_size
+        attr_accessor :performance_id, :scenario_id, :vars, :cookie
+        def size
+            @size
+        end
+        def init
+            @performance_id = (@scenario_id.to_s + sprintf(@tmpl, @loop)).to_i
+            act = @commands.shift
+            act.performance_id = @performance_id
+            @request_pool.add(act)
+        end
+        def next
+            @action_counter -= 1
+            #puts "action: #{@action_counter}"
+            #puts "loop: #{@loop}"
+            if @action_counter == 0
+                @loop -= 1
+                #これで最後
+                return if @loop == 0
+                @action_counter = @actions_size
+                @performance_id = (@scenario_id.to_s + sprintf(@tmpl, @loop)).to_i
+            end
+            if @commands.empty?
+                #ループが残っていればまた最初から
+                @commands = @commands_org.clone
+                @cookie = {}
+                @vars   = {}
+            end
+            act = @commands.shift
+            act.performance_id = @performance_id
+            @request_pool.add(act)
+        end
+        def each &block
+            @commands_org.each do |cm|
+                block.call(cm)
+            end
+        end
+        def create_scenario
+            @commands = []
+            @performance_id = (@scenario_id.to_s + sprintf(@tmpl, @loop)).to_i
+            @actions_size = @config['actions'].inject(0){|sum, i|
+                times = i['times']
+                sum + times
+            }
+            baseconfig = @config.clone
+            baseconfig.delete('actions')
+            1.upto(@page_size){
+                @config["actions"].each do |actconfig|
+                    actconfig = baseconfig.merge(actconfig)
+                    times = actconfig['times']
+                    command = actconfig['class_obj']
+                    1.upto(times) do
+                        actconfignew = actconfig.clone
+                        obj = command.new(actconfignew, self)
+                        obj.set_wait if(actconfignew['wait'])
+                        @commands << obj
+                    end
+                end
+            }
+            @action_counter = @actions_size
+            @commands_org = @commands.clone
+            @size = @actions_size * @loop_org
+        end
+        def set_cookie cookie
+            return unless cookie
+            k, v = cookie[0...cookie.index(';')].split('=')
+            @cookie[k] = v if k && v
+        end
+        def create_act idx
+            actconf = @actconfigs[idx]
+            obj = actconf['class_obj']::new(actconf)
+            obj.set_scenario(self)
+            obj
+        end
+    end
+end

data/lib/internethakai/scenario.tmpl

@@ -0,0 +1,58 @@
+loop:
+  1
+max_request:
+  1
+max_scenario:
+  1
+log_level:
+  2
+encoding: Shift_JIS
+ranking: 20
+timeout: 5
+show_report: true
+save_report: false
+
+## 負荷試験対象のサイトに変更してください
+domain:
+  "<%= domain %>"
+
+
+<% if social %>
+## ソーシャルアプリの場合コメントアウト
+client_handler: "SocialClientHandler"
+opensocial_id_path: #ユーザーidを書いたファイルを用意してください
+  ./userids
+opensocial_app_id:
+  yourapp
+<% else %>
+## ソーシャルアプリの場合コメントアウト
+#client_handler: "SocialClientHandler"
+#opensocial_id_path: #ユーザーidを書いたファイルを用意してください
+#  ./userids
+#opensocial_app_id:
+#  app
+<% end %>
+
+## ユーザーエージェントを変更したい場合、コメントアウト
+#user_agent:
+#  'Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)'
+
+
+## fork して使用する場合コメントアウト
+#max_process: 4 
+
+
+#以下にアクション(シナリオ)を書きます
+actions:
+   - 
+     path: /
+## example
+#  -
+#    path: /somepath
+#  -
+#    path: /somepath
+#    method: POST
+#    post_params:
+#      key1: hoge
+#      key2: fuga
+    

data/lib/internethakai/scenario_builder.rb

@@ -0,0 +1,183 @@
+module InternetHakai
+    class ScenarioBuilder < BaseHandler
+        UNIQUE_BY_THREAD = false
+        require 'socket'
+        require 'yaml'
+        DEFAULT_CONFIG = {
+            "loop" => 1,
+            "rev" => true,
+            "max_scenario" => 1,
+            "max_request" => 1,
+            "log_level" => 3,
+            "ranking" => 5,
+            "follow_redirect" => true,
+            "queue" => "SimpleQueue",
+            "action_handler" => "RequestAction",
+            "client_handler" => "ClientHandler",
+            "response_handler" => "TimeRegister",
+            "request_pool" => "RequestPool",
+            "http_client" => "HttpClient",
+            "scenario_executer" => "ScenarioExecuter",
+            "logger" => "Logger",
+            "encoding" => "Shift_JIS",
+            "save_report" => false,
+            "reporter" => "TsvReporter",
+            "show_report" => false,
+            "sleep" => false,
+            "timeout" => 10,
+            "actions" => [],
+        }
+        def get_config
+            @config
+        end
+        def prepare_config
+            baseconfig = @config
+            basehost, baseport, = Util::parse_url(baseconfig['domain'])
+            basehostaddress = IPSocket::getaddress(basehost)
+            baseconfig['host'] = basehostaddress
+            baseconfig['host_name'] = basehost unless baseconfig['host_name']
+            baseconfig['port'] = baseport
+            baseconfig['encoding'] ||= 'UTF-8'
+
+            #後方互換性のため
+            if baseconfig.has_key?('fork') && baseconfig['fork'].has_key?('max_process') && baseconfig['fork']['max_process'] > 1 && !baseconfig.has_key?('max_process')
+                baseconfig['max_process'] = baseconfig['fork']['max_process']
+            else
+                baseconfig['max_process'] ||= 1
+            end
+            #後方互換性のため
+            if baseconfig.has_key?('concurrency') && baseconfig['max_scenario'] == 1
+                baseconfig['max_scenario'] = baseconfig['concurrency']
+            end
+
+            headers = baseconfig.has_key?('header') ? baseconfig['header'] : Hash::new
+            headers['User-Agent'] = baseconfig['user_agent'] if baseconfig.has_key?('user_agent')
+            headers['Host'] = baseconfig['host_name']
+            baseconfig['header'] = headers
+            hosts = []
+            hosts << [basehostaddress, basehost, baseport]
+            hosts_cache = {}
+            hosts_cache[hosts[0].join(':')] = true
+            idx = 0
+
+            base = baseconfig.clone
+            base.delete('actions')
+            actions = []
+            @config["actions"].each do |actconfig|
+                if actconfig.is_a? String
+                    path = actconfig
+                    actconfig = Hash::new
+                    actconfig['path'] = path
+                end
+                url = actconfig['path']
+                actconfig['path'] = url
+                actconfig['method'] = 'GET' unless actconfig.has_key? 'method'
+                actconfig['class'] = baseconfig['action_handler'] unless actconfig.has_key? 'class'
+                classname = actconfig["class"]
+                command = BaseHandler::get_class(classname)
+                raise "invalid class" unless command < BaseAction
+                actconfig['class_obj'] = command
+
+                if (!url.match( /^http:\/\// ))
+                    url = baseconfig['domain'].to_s + url.to_s
+                    actconfig['host'] = basehostaddress
+                    actconfig['host_name'] = basehost
+                    actconfig['port'] = baseport
+                else
+                    host, port, path = Util::parse_url(url)
+                    #名前解決
+                    address = IPSocket::getaddress(host)
+                    actconfig['host'] = address
+                    actconfig['path'] = path
+                    actconfig['host_name'] = host unless baseconfig['host_name']
+                    actconfig['port'] = port
+                    info = [address, host, port]
+                    headers = actconfig.has_key?('header') ? actconfig['header'] : Hash::new
+                    headers['User-Agent'] = baseconfig['user_agent'] if baseconfig.has_key?('user_agent')
+                    headers['Host'] = actconfig['host_name']
+                    actconfig['header'] = headers
+                    key = info.join(':')
+                    unless hosts_cache.has_key?(key)
+                        hosts << info
+                        hosts_cache[key] = true
+                    end
+                end
+                if(actconfig.has_key?('post_params'))
+                    actconfig['post_string'] = Util::hash2poststring(actconfig['post_params'])
+                end
+                unless actconfig.has_key?('type')
+                    actconfig['type'] = actconfig['path'].gsub( /\?.*$/, '' ).gsub( /[#;].*/, '' )
+                end
+                if actconfig.has_key?('expr')
+                    if RUBY_VERSION >= "1.9"
+                        s = actconfig['expr'].encode(baseconfig['encoding'])
+                        actconfig['compiled_expr'] = Regexp::compile(s)
+                    else
+                        actconfig['compiled_expr'] = Regexp::compile(actconfig['expr'])
+                    end
+                end
+                actconfig['url'] = url
+                time = actconfig['times'] || 1
+                actconfig['times'] = time unless actconfig.has_key?('times')
+                idx += 1
+                actions << actconfig
+            end
+            scenario_size = actions.inject(0){|sum, i|
+                sum + i['times']
+            }
+            @config['scenario_size'] = scenario_size
+            @config['actions'] = actions
+            @config['hosts'] = hosts
+            #concurrency はデフォルトでスレッドと同じ
+            if !@config.has_key?('max_scenario') || @config['max_scenario'] < @config['max_request']
+                @config['max_scenario'] = @config['max_request'] 
+            end
+            @config['max_scenario'] = 1 if @config['max_scenario'] <= 0
+
+            @config['max_scenario_show'] = @config['max_scenario']
+            @config['max_request_show'] = @config['max_request']
+            if @config['max_process'] > 1
+                #fork する場合は、concurrency をプロセス数で割る
+                process = @config['max_process']
+                @config['max_process'] = process
+                @config['max_scenario'] = (@config['max_scenario'] / process).round
+                @config['max_scenario'] = 1 if @config['max_scenario'] <= 0
+                @config['max_request'] = (@config['max_request']/ process).round
+                @config['max_request'] = 1 if @config['max_request'] <= 0
+            end
+            ch = BaseHandler::get_class(@config["client_handler"])
+            @config = ch::on_config_load(@config)
+        end
+        def load path
+            begin
+                basedir = File::expand_path(File::dirname(path))
+                str = File::open(path){|io|io.read}
+                load_config(str, basedir)
+            rescue
+                raise $!
+            end
+        end
+        def load_config str, basedir='.'
+            default_config = DEFAULT_CONFIG
+            if(RUBY_VERSION.to_f >= 1.9)
+                str.force_encoding('UTF-8')
+            end
+            if str.include?('%%BASEDIR%%')
+                puts "WARNING: %%BASEDIR%% is obsolete"
+                str.gsub!('%%BASEDIR%%', basedir)
+            end
+            @config = ::YAML::load(str)
+            raise "invalid file" unless @config
+            @config = default_config.merge(@config)
+            prepare_config
+            if(@config["require"])
+                begin
+                    require @config["require"]
+                rescue
+                    @logger.run("class load error: #{@config['require']}", 1)
+                end
+            end
+            @config
+        end
+    end
+end