internethakai 0.2.0

data/lib/internethakai/client_handler.rb

@@ -0,0 +1,175 @@
+module InternetHakai
+    # = httpクライアントをいじるクラス
+    class ClientHandler
+        UNIQUE_BY_THREAD = true
+        def self::on_config_load config
+            config['actions'] = config['actions'].map do |actconfig|
+                actconfig['usr_var'] = false
+                if(/%\(.*\)%/ =~ actconfig['path'])
+                    actconfig['use_var'] = true
+                end
+                if(actconfig.has_key?('post_string') && /%25\(.*\)%25/ =~ actconfig['post_string'])
+                    actconfig['use_var'] = true
+                end
+                actconfig
+            end
+            config
+        end
+        @@vars = Hash::new
+        attr_accessor :vars, :performance_id
+        def initialize scenario
+            @scenario = scenario
+        end
+        def send method, host, port, path
+            @client = get_client unless @client
+        end
+        def set_opt opt
+            @opt = opt
+        end
+        def set_var key, val
+            @scenario.vars[key] = val
+        end
+        def extend_vars str
+            str = str.to_s.clone
+            for k,v in @scenario.vars
+                str.gsub!("%(#{k})%", v)
+            end
+            str
+        end
+        def extend_vars2 str
+            str = str.to_s.clone
+            for k,v in @scenario.vars
+                str.gsub!("%25(#{k})%25", v)
+            end
+            str
+        end
+        def handle_url client, url, opt
+            opt['use_var'] ? extend_vars(url) : url
+        end
+        def handle_body client, body, opt
+            opt['use_var'] ? extend_vars2(body) : body
+        end
+        def handle_client client
+            @client = client
+        end
+    end
+
+    # = ケータイサイト用
+    class MobileClientHandler < ClientHandler
+        DOCOMO = 1
+        KDDI = 2
+        SOFTBANK = 3
+        def self::on_config_load config
+            if(config.has_key? "uid_path")
+                s = File::open(config["uid_path"]){|io|io.read}
+                lines = s.split("\n")
+                config["uids"] = lines.map{|i|i.split(",")}.sort_by{rand}
+            end
+            config
+        end
+        def handle_client client
+            super
+            uid = get_uid
+            case get_carrier
+                when DOCOMO
+                @client.set_header("http_x_dcmguid", uid)
+                when KDDI
+                @client.set_header("http_x_up_subno", uid)
+                when SOFTBANK
+                @client.set_header("http_x_jphone_uid", uid)
+            end
+            @client
+        end
+        def handle_url client, url, opt
+            url = super
+            q = Util::get_query(url)
+            if q["uid"] == "NULLGWDOCOMO"
+                url = Util::set_params(url.gsub("uid=NULLGWDOCOMO", ""), {"uid" => get_uid})
+            end
+            url
+        end
+        def get_uid
+            ids = @opts["uids"]
+            return ids unless ids.is_a? Array
+            idx = @performance_id % ids.size
+            ids[idx][1]
+        end
+        def get_carrier
+            ids = @opts["uids"]
+            idx = @performance_id % ids.size
+            data = ids[idx]
+            unless data
+                if @opt["carrier"]
+                    @opt["carrier"]
+                end
+                DOCOMO
+            else
+                idx = @performance_id % ids.size
+                ids[idx][0]
+            end
+        end
+    end
+
+    # = ソーシャルアプリ用
+    class SocialClientHandler < ClientHandler
+        def self::on_config_load config
+            super
+            if(config.has_key? "opensocial_id_path")
+                s = File::open(config["opensocial_id_path"]){|io|io.read}
+                lines = s.split("\n")
+                config["opensocial_ids"] = lines.map{|i|i.split("\t")}.sort_by{rand}
+                config['opensocial_id_len'] = config['opensocial_ids'][0].size
+            elsif config['opensocial_ids']
+                r = []
+                config['opensocial_ids'].each do |i|
+                    r << [i] unless i.is_a? Array
+                end
+                config['opensocial_ids'] = r
+                config['opensocial_id_len'] = config['opensocial_ids'][0].size
+            else
+                config['opensocial_ids'] = [1]
+            end
+            if !config['opensocial_app_id']
+                config['opensocial_app_id'] = 'app'
+            end
+            config['opensocial_app_id'] = config['opensocial_app_id'].to_s
+            config
+        end
+        def handle_client client
+            super
+            @client
+        end
+        def handle_body  client, body, opt
+            body = super
+            if body[body.size-1] != 38
+                body += "&"
+            end
+            id = get_opensocial_id
+            app_id = @opt['opensocial_app_id']
+            body += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
+        end
+        def handle_url client, url, opt
+            url = super
+            if ! url.include? "?"
+                url += "?"
+            elsif url[url.size-1] == 63
+                url
+            elsif url[url.size-1] != 38
+                url += "&"
+            end
+            id = get_opensocial_id
+            #app_id = @opt['opensocial_app_id']
+            #url += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
+            url += 'opensocial_app_id=' + @opt['opensocial_app_id'] + '&opensocial_owner_id=' + id + '&opensocial_viewer_id=' + id
+        end
+        def get_opensocial_id
+            ids = @opt["opensocial_ids"]
+            id = ids[@performance_id % ids.size]
+            1.upto(@opt['opensocial_id_len']-1) do |idx|
+                set_var(idx, id[idx])
+            end
+            r, = id
+            r.to_s
+        end
+    end
+end

data/lib/internethakai/client_queue.rb

@@ -0,0 +1,27 @@
+module InternetHakai
+    class ClientQueue
+        def initialize size
+            @queue = []
+            @size = size
+            @ptr = size
+        end
+        def empty?
+            @ptr == 0
+        end
+        def get
+            if @ptr == 0
+                nil
+            else
+                @ptr -= 1
+                @queue.shift
+            end
+        end
+        def collect cl
+            @ptr += 1
+            @queue << cl
+        end
+        def add cl
+            @queue << cl
+        end
+    end
+end

data/lib/internethakai/concurrency_manager.rb

@@ -0,0 +1,333 @@
+module InternetHakai
+    class ConcurrencyManager
+        require 'thread'
+        def initialize
+            @fork_mode = false
+            @rev_mode = false
+            @process = 0
+            @suffix = ''
+            @mutex = nil
+        end
+        attr_accessor :suffix, :process
+        def prepare_rev
+            return if @rev_mode #2回実行しないように
+            require 'rubygems'
+            require 'rev'
+            require 'internethakai/hakairev'
+            #TimeRegistProcess::run
+            @config['http_client'] = 'RevHttpClient'
+            @config['scenario_executer'] = 'NonBlockScenarioExecuter'
+            @config['request_pool'] = 'RequestPoolNonThread'
+            @config['queue'] = 'TaskQueue'
+            #@config['response_handler'] = 'PipeTimeRegister'
+            @config['response_handler'] = 'RevTimeRegister'
+            @config['scan']
+            @rev_mode = true
+        end
+        def prepare_fork
+            return if @fork_mode
+            @concurrency = @config['max_scenario']
+            @process_max = @config['max_process'] || 2
+            @fork_mode = true
+       end
+        def run config, basepath, starttime
+            @config = config
+            @scenario_handler = BaseHandler::get_handler("ScenarioHandler")
+            @failure = false
+            @basepath = basepath
+            BaseHandler::set_config(@config)
+            @logger = BaseHandler::get_handler(@config["logger"])
+            @threads = {}
+
+            @concurrency = @config['max_scenario']
+            if @config['rev']
+                prepare_rev
+            end
+            if @config['max_process'] > 1
+                prepare_fork
+            end
+            @register = BaseHandler::get_handler(@config['response_handler'])
+            reporter = BaseHandler::get_handler(@config["reporter"])
+
+            digit = @config['max_request'].to_s.size
+            @id_tmpl = "%0#{digit}d"
+            if @config['max_process'] > 1
+                run_fork
+            else
+                run_process
+            end
+        end
+        def run_process
+            #並列数だけシナリオのコピーをつくっておく
+            @config['request_pool'] = 'RequestPoolNonThread' if @config["max_request"] <= 1
+            @scenarios = []
+            1.upto(@concurrency) do
+                @scenarios << @scenario_handler.create_scenario(@suffix)
+            end
+            @scenario_handler.on_scenarios_created
+            $REQUEST_COUNT = 0
+            begin
+                if @rev_mode
+                    rev_start
+                elsif @config["max_request"] <= 1
+                    single_start
+                else
+                    thread_start
+                end
+            rescue SystemExit
+                return
+            rescue Exception => e
+                raise e
+                puts "error: #{e.message}"
+                puts e.backtrace[0]
+                if @fork_mode
+                    #死ぬ前に親に連絡
+                    Process::kill(:USR1, Process::ppid)
+                else
+                    report_collect
+                end
+            end
+        end
+        def single_start
+            th_cnt = 1
+            @threads[1] = Thread::current
+            @scenarios.each do |sc|
+                sc.init
+            end
+            #シングルスレッドのときは先にプールにためておく
+            run_executer th_cnt
+            if @failure
+                page_check
+                exit 1
+            else
+                exit 0
+            end
+        end
+        def page_check
+            keys = @result_record.keys
+            pages = []
+            keys.each do |key|
+                err = @result_record[key][:errorcount] || 0
+                if(err > 0)
+                    pages << key
+                end
+            end
+            unless pages.empty?
+                @logger.run("ERROR URL: #{pages.join(',')}\n", 2)
+            end
+        end
+        def thread_start
+            @mutex = Mutex::new
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+            @cv = ConditionVariable::new
+
+            @scenarios.each do |sc|
+                sc.init
+            end
+            Thread.abort_on_exception 
+            @scenario_handler.set_on_turn_complete &method(:on_complete)
+            1.upto(@config["max_request"]) do |th_cnt|
+                @threads[th_cnt] = Thread::new(th_cnt) do |th_cnt|
+                    run_executer th_cnt
+                end
+            end
+            @threads[1].join
+            @threads[:main] = Thread::current
+            @mutex.synchronize{
+                @cv.wait(@mutex)
+            }
+            @threads.each do |k,th|
+                th.kill if th!=Thread::current && k!=:main
+            end
+            if @failure
+                exit 1
+            else
+                exit 0
+            end
+        end
+        def run_executer th_cnt
+            th_id = (sprintf(@id_tmpl, th_cnt) + @suffix).to_i
+            report = BaseHandler::get_handler(@config["response_handler"])
+            Thread.current[:result] = report
+            BaseHandler::set_thread_id(th_id)
+            cls = BaseHandler::get_class(@config['scenario_executer'])
+            @scenario_handler.set_on_turn_complete(&@register.method(:exec))
+            exec = cls::new
+            exec.set_on_complete &method(:on_complete)
+            @starttime = Time::now
+            exec.run
+        end
+        # = 1プロセス終了時に呼ばれる
+        def on_complete
+            total = Time::now - @starttime
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+            @request_pool.close
+            #puts "thread collect start"
+            thread_collect
+            #puts "thread collect end"
+            report_collect
+            #puts "report collect end"
+            @logger.run("TotalTime: " + total.to_s + "\n", 2) unless @fork_mode
+            @failure = true if $error
+            @cv.signal if @cv
+        end
+        def create_id loop_c, th_c
+            (loop_c.to_s + sprintf(@id_tmpl, th_c) + @suffix).to_i
+        end
+        def thread_collect
+            count = 0
+            status = '.'
+            flag = false
+            count = 0
+            @threads.each do |k, th|
+                if(th[:result])
+                    status = 't'
+                    ect = 0
+                    th[:result].get_record.each do |name,res|
+                        ect += 1 if res[:errorcount] > 0
+                    end
+                    if ect > 0
+                        status = 'e'
+                        flag = true unless flag
+                    end
+                    #th.kill if th!=Thread::current && k!=:main
+                end
+                #@logger.run status, 2
+            end
+            @logger.run "\n", 2 unless @fork_mode
+            @logger.run "result: ", 2 unless @fork_mode
+            if flag
+                @logger.run "failure\n", 2 unless @fork_mode
+            else
+                @logger.run "success\n", 2 unless @fork_mode
+            end
+            @failure = flag
+        end
+        def report_collect
+            d = (Time::now - @starttime)*1000
+            if @fork_mode && @forkdb
+                @forkdb.transaction do
+                    if not @forkdb['time']
+                        @forkdb['time'] = d
+                    elsif @forkdb['time'] < d
+                        @forkdb['time'] = d
+                    end
+                    @forkdb['request_count'] = 0 unless @forkdb['request_count']
+                    @forkdb['request_count'] += $REQUEST_COUNT
+                end
+            else
+                @logger.run "\nTimePerRequest: #{d / ($REQUEST_COUNT / @config['max_request'])}\n", 2
+                @logger.run "RequestPerSec: #{$REQUEST_COUNT / (d/1000)}\n", 2
+            end
+            record = ResponseRecord::new
+            @threads.each do |k, th|
+                if th[:result]
+                    record += th[:result].get_record
+                end
+            end
+            @logger.run("\n\n", 2) unless @fork_mode
+            reporter = BaseHandler::get_handler(@config["reporter"])
+            reporter.run(record)
+            @result_record = record
+        end
+        def run_fork
+            raise "invalid config: fork: nil" unless @config['fork']
+            org_reporter = @config['reporter']
+
+            @logger.run("Process: #{@process_max}\n", 2)
+            @parent = false
+
+            flag = false
+            Signal::trap(:USR1){
+                return if flag
+                flag = true
+                #parent
+                puts "error: SIGUSR1"
+                @pids.each do |pid|
+                    Process::kill(:USR2, pid)
+                end
+            }
+            Signal::trap(:INT){
+                flag = true
+                puts "error: SIGINT"
+                @pids.each do |pid|
+                    Process::kill(:USR2, pid)
+                end
+            }
+            Signal::trap(:USR2){
+                #child
+                report_collect
+                exit
+            }
+
+            #各プロセスにはpstoreファイルに集計させる
+            @config['reporter'] = 'PStoreReporter'
+            fname = 'tmp.db'
+            fpath = File::join(@basepath, fname)
+            if FileTest::exists?(fpath)
+                raise 'cannot create db' unless FileTest::writable?(fpath)
+                raise 'cannot create db' unless File::delete(fpath)
+            end
+            db = PStore::new(fpath)
+            @forkdb = db
+
+            reporter = BaseHandler::get_handler(@config["reporter"])
+            reporter.set_dir(@basepath)
+            reporter.init_filename(fname)
+
+            begin
+                @pids = []
+                @process = @process_max
+                digit = @process_max.to_s.size
+                tmpl = "%0#{digit}d"
+                1.upto(@process_max) do |i|
+                    @pids << Process::fork {
+                        @suffix = sprintf(tmpl, i)
+                        run_process
+                    }
+                    #1プロセス目以外は出力を少なく
+                    @scenario_handler.silence if i==2
+                end
+                @forkreport = false
+                @org_fork_reporter = org_reporter
+                results = Process::waitall
+                failure = false
+                results.each do |r|
+                    if r[1].to_i != 0
+                        failure = true
+                        break
+                    end
+                end
+                @logger.run "\n", 2
+                @logger.run "result: ", 2
+                unless failure
+                    @logger.run "success\n", 2
+                else
+                    @logger.run "failure\n", 2
+                end
+                fork_report
+            ensure
+                fork_report unless @forkreport
+                File::delete(fpath) if File::exists?(fpath)
+            end
+        end
+        def fork_report
+            @forkreport = true
+            require 'pstore'
+            result = nil
+            time = nil
+            count = nil
+            @forkdb.transaction do
+                result = @forkdb['result']
+                time = @forkdb['time']
+                count = @forkdb['request_count']
+            end
+            @logger.run "\nTimePerRequest: #{time / (count / @config['max_request_show'])}\n", 2 if time && count
+            @logger.run "RequestPerSec: #{count / (time/1000)}\n", 2 if time && count
+
+            reporter = BaseHandler::get_handler(@org_fork_reporter)
+            reporter.run(result)
+            reporter = nil
+        end
+    end
+end