internethakai 0.2.0

data/lib/internethakai/executor.rb

@@ -0,0 +1,129 @@
+module InternetHakai
+    class RequestPool < BaseHandler
+        UNIQUE_BY_THREAD = false
+        require 'thread'
+        def initialize id
+            @queue = Queue::new
+            @enable = true
+            @closed = false
+        end
+        def clear
+            @queue.clear
+            @queue = Queue::new
+        end
+        def add action
+            return unless @enable
+            @queue.enq(action)
+        end
+        def empty?
+            @queue.empty?
+        end
+        def get
+            return if @closed
+            r = @queue.deq
+            if ! @enable and @queue.empty?
+                #enable=false で キューが空になった時点でclose
+                close
+            end
+            return r
+        end
+        def close
+            @closed = true
+            1.upto(@queue.num_waiting) do
+                @queue.enq(-1)
+            end
+            @queue.clear
+        end
+        def quit
+            @enable = false
+            #enable=false で キューが空になった時点でclose
+            close if @queue.empty?
+        end
+    end
+    class SimpleQueue < BaseHandler
+        UNIQUE_BY_THREAD = false
+        def add *ar
+            task, arg = ar
+        end
+    end
+    class RequestPoolNonThread < BaseHandler
+        UNIQUE_BY_THREAD = false
+        def initialize id
+            @queue = []
+            @closed = false
+            @enable = true
+        end
+        def clear
+            @queue = []
+        end
+        def empty?
+            @queue.empty?
+        end
+        def add action
+            return unless @enable
+            @queue << action
+        end
+        def get
+            return -1 if @closed
+            if ! @enable and @queue.empty?
+                close
+                return -1
+            end
+            @queue.shift
+        end
+        def close
+            @closed = true
+        end
+        def quit
+            @enable = false
+            close if @queue.empty?
+        end
+    end
+    # = シナリオを実行するクラス
+    class ScenarioExecuter
+        def initialize
+            @scenario_handler = BaseHandler::get_handler("ScenarioHandler")
+            @enable = true
+            @logger = BaseHandler::get_handler('Logger')
+            @config = BaseHandler::get_config
+            @max_request = @config['max_request']
+            @register = BaseHandler::get_handler(@config['response_handler'])
+            client_cls =  BaseHandler::get_class(@config['http_client'])
+            cl = client_cls::create(@config['host'], @config['port'])
+            cl.timeout = @config['timeout']
+            cl.set_headers(@config['header'])
+            @client = cl
+        end
+        # すべてのシナリオの終了時に実行されます
+        def set_on_complete &block
+            @on_complete_callback = block
+        end
+        def on_complete
+            return if @scenario_handler.closed?
+            if @on_complete_callback
+                @scenario_handler.close
+                @on_complete_callback.call
+            end
+        end
+        def run
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+            pool = @request_pool
+            client = @client
+            scenario_handler = @scenario_handler
+            register = @register
+            count = 0
+            while true
+                break unless @enable
+                if scenario_handler.complete?
+                    on_complete
+                end
+                act = pool.get
+                break if act.nil? or act==-1
+                act.set_client(client)
+                act.run
+                register.exec
+            end
+            on_complete
+        end
+    end
+ end

data/lib/internethakai/generator.rb

@@ -0,0 +1,32 @@
+module InternetHakai
+    class Generator
+        require 'erb'
+        @@readline = require 'readline'
+        def build
+            s = open(File::join(File::dirname(__FILE__), 'scenario.tmpl')){|io|io.read} 
+            template = ERB::new(s)
+            puts "your site\n    ex. http://example.com"
+            if @@readline
+                domain = Readline.readline("> ", true)
+            else
+                domain = gets
+            end
+            if(domain)
+                domain.chomp!
+                domain = 'http://' + domain unless /^https?:\/\// =~ domain
+            else
+                domain = 'http://example.com'
+            end
+            social = false
+            data = template.result(binding)
+            fname = 'scenario'
+            idx = 0
+            while(File::exists?(fname+"-#{idx}.yml"))
+                idx += 1
+            end
+            fname = fname + "-#{idx}.yml"
+            open(fname, 'w'){|io|io.write(data)}
+            puts "save to #{fname}"
+        end
+    end
+end

data/lib/internethakai/hakairev.rb

@@ -0,0 +1,119 @@
+require 'internethakai/hakairev/executor'
+require 'internethakai/hakairev/http_client'
+require 'internethakai/hakairev/task'
+require 'internethakai/hakairev/time_register'
+require 'internethakai/hakairev/timer_factory'
+require 'internethakai/hakairev/revpipe'
+require 'internethakai/hakairev/monkey'
+
+module InternetHakai
+    class ConcurrencyManager
+        def rev_executer
+            @register.exec
+            @scenario_executer.run_one_action
+        end
+        def rev_on_complete
+            @queue.clear
+            on_complete
+            rev_stop
+        end
+        def rev_scenario_push
+            flag = true
+            @scenarios.each do |sc|
+                flag = sc.next
+            end
+            unless flag
+                #@logger.run("all scenario push\n", 2)
+                @request_pool.quit
+                @threads[:scenario].kill if @threads[:scenario]
+            end
+            flag
+        end
+        def start_task_thread
+            q = @queue
+            Thread::start(q) do |q|
+                while @rev_running
+                    q.run
+                end
+            end
+        end
+        def rev_start
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+            th_cnt = 1
+            th_id = (sprintf(@id_tmpl, th_cnt) + @suffix).to_i
+            BaseHandler::set_thread_id(th_id)
+            @queue = BaseHandler::get_handler('TaskQueue')
+            TimerFactory::new(@config['timeout'], @config['max_request'], Rev::Loop::default)
+            @client_queue = ClientQueue::new(@config['max_request'])
+            1.upto(@config['max_request']) do |i|
+                client_cls =  BaseHandler::get_class(@config['http_client'])
+                cl = client_cls::create(@config['host'], @config['port'])
+                cl.timeout = @config['timeout']
+                cl.set_headers(@config['header'])
+                cl.tfactory = TimerFactory::instance
+                cl.client_queue = @client_queue
+                @client_queue.add(cl)
+            end
+
+            Thread::current[:heartbeat] = Time::now
+            report = BaseHandler::get_handler(@config["response_handler"])
+            Thread.current[:result] = report
+
+            @scenarios.each do |sc|
+                #最初のアクションだけセット
+                #sc.each do |act|
+                #    act.client_queue = @client_queue
+                #end
+                sc.init
+            end
+
+            @threads[:main] = Thread::current
+            @starttime = Time::now
+            @scenario_handler.set_on_complete(&method(:rev_on_complete))
+            @scenario_handler.set_on_turn_complete(&@register.method(:exec))
+            #rev_executer_run
+            @rev_running = true
+            #start_task_thread
+            rev_loop_run
+            if @failure
+                exit 1
+            else
+                exit 0
+            end
+        end
+        def rev_loop_run
+            loop = Rev::Loop::default
+            q = @queue
+            rq = @request_pool
+            clq = @client_queue
+            while @rev_running
+                flag = false
+                if loop.has_active_watchers?
+                    loop.run_nonblock
+                end
+                if !rq.empty? and !clq.empty?
+                    a = rq.get
+                    cl = clq.get
+                    #a.client_queue = clq
+                    #puts "set: #{cl.object_id}"
+                    a.set_client(cl)
+                    a.run
+                    flag = true
+                end
+                if loop.has_active_watchers?
+                    loop.run_nonblock
+                end
+                if !q.empty?
+                    q.run
+                    flag = true
+                end
+                if !flag
+                    loop.run_once
+                end
+            end
+        end
+        def rev_stop
+            @rev_running = false
+        end
+    end
+end

data/lib/internethakai/hakairev/executor.rb

@@ -0,0 +1,43 @@
+module InternetHakai
+    class NonBlockScenarioExecuter < ScenarioExecuter
+        def initialize
+            super
+            @request_pool = BaseHandler::get_handler(@config['request_pool'])
+            @concurrency = @config['concurrency']
+            @max_request = @config['max_request']
+            @request_counter = @max_request
+            @acts = []
+            @timer = nil
+            @queue = BaseHandler::get_handler('TaskQueue')
+            @method_dispatch_next = method(:dispatch_next)
+            @method_run_one = method(:run_one)
+        end
+        attr_accessor :timer
+        #max_request分のactionが終了
+        def on_action_complete &block
+            if block
+                @on_action_complete_callback = block
+            elsif @on_action_complete_callback
+                @on_action_complete_callback.call
+            end
+        end
+        def dispatch_next
+            @queue.add(@method_run_one)
+        end
+        def run_one
+            act = @request_pool.get
+            #puts "act: #{act.class} :#{act.object_id}" if act!=-1 && act
+            if act.nil?
+                dispatch_next
+                return
+            elsif act == -1
+                    #-1が返ったらおわり
+                #dispatch_next
+                return
+            end
+            #actionのcallbackに次の実行を設定
+            act.set_on_complete(&@method_dispatch_next) unless act.has_on_complete?
+            act.run
+        end
+    end
+end

data/lib/internethakai/hakairev/http_client.rb

@@ -0,0 +1,362 @@
+module InternetHakai
+    class RevHttpConnection < Rev::HttpClient
+        def initialize args
+            @remote_addr, @remote_port = args
+            soc = TCPConnectSocket::new(::Socket::AF_INET, @remote_addr, @remote_port)
+            super(soc)
+        end
+        def init
+            @content = ''
+            @writable_flag = false
+            @send_flag = false
+            @busy = false
+            @on_request_complete = nil
+            @requested = false
+            @response_object = InternetHakai::ResponseObject::new
+            self
+        end
+        attr_reader :connected, :busy
+        def on_read s
+            super
+        end
+        def on_readable
+            begin
+                on_read @_io.read_nonblock(INPUT_SIZE)
+            rescue Errno::EAGAIN
+            rescue Errno::ECONNRESET, EOFError
+            rescue IOError => e
+                @exception = e
+                on_error('io error')
+            rescue Exception => e
+                @exception = e
+                on_error('io error')
+            end
+        end
+        def reconnect
+            @requested = false
+            @busy = false
+            free
+            socket = TCPConnectSocket::new(::Socket::AF_INET, @remote_addr, @remote_port)
+
+            @_io = socket
+            #@_write_buffer  ||= ::IO::Buffer.new
+            @_read_watcher  = Watcher.new(socket, self, :r)
+            @_write_watcher = Watcher.new(socket, self, :w)
+            @address_family, @remote_port, @remote_host, @remote_addr = socket.peeraddr
+
+
+            @state = :response_header
+            #@data = ::IO::Buffer.new
+
+            @response_header = Rev::HttpResponseHeader.new
+            @chunk_header = Rev::HttpChunkHeader.new
+            #initialize(socket)
+
+            @_connector = Connector::new(self, socket)
+            @_connector.attach(@loop)
+        end
+        def close
+            super
+        end
+        def on_writable
+            @writable_flag = true unless @writable_flag
+            # writable + send 両方そろったら開始時間
+            if @send_flag && !@starttime
+                @starttime = Time::now
+            end
+            begin
+                @_write_buffer.write_to(@_io)
+
+            rescue Errno::EPIPE, Errno::ECONNRESET => e
+            rescue IOError => e
+            rescue Exception => e
+                @exception = e
+                on_error('io error')
+            end
+            if @_write_buffer.empty?
+                disable_write_watcher
+                on_write_complete
+            end
+        end
+        def send_request
+            @send_flag = true
+            @starttime = Time::now
+            super
+        end
+        def attached?
+            if @_connector
+                @_connector.attached?
+            elsif @_read_watcher
+                @_read_watcher.attached?
+            elsif @_resolver
+                @_resolver.attached
+            else
+                super
+            end
+        end
+        def on_body_data(data)
+            @content += data
+        end
+        def on_request_complete
+            @response_time = Time::now - @starttime
+            self.close
+        end
+        def on_connect_failed
+            @exception = SocketError::new('connection error')
+            on_error('')
+        end
+        def on_resolve_failed
+            @exception = SocketError::new('dns error')
+            on_error('')
+        end
+        def on_failure
+            @exception = SocketError::new
+            on_error('')
+        end
+        def on_error(reason='error')
+            unless @starttime.nil?
+                @response_time = Time::now - @starttime
+            else
+                @response_time = 0
+            end
+            close if @_io and !@_io.closed?
+            @requested = false
+            @exception = reason if @exception.nil?
+            @busy = false
+        end
+        def to_response_object
+            r = @response_object
+            r.body = @content.to_s
+            r.status = @response_header.http_status.to_i
+            r.content_type = @response_header["CONTENT_TYPE"].to_s
+            r.location = @response_header["LOCATION"]
+            r.cookie   = @response_header['SET_COOKIE']
+            r.time = @response_time
+            r
+        end
+        def attach loop
+            @loop = loop unless @loop
+            if @_connector
+                @_connector.attach loop
+            elsif @_read_watcher
+                @_read_watcher.attach loop
+            elsif @_resolver
+                @_resolver.attache loop
+            else
+                return
+            end
+        end
+=begin
+        def parse_response_header
+            return false unless parse_header(@response_header)
+
+            unless @response_header.http_status and @response_header.http_reason
+                on_error "no HTTP response"
+                @state = :invalid
+                return false
+            end
+
+            on_response_header(@response_header)
+
+            @state = :body
+            @bytes_remaining = @response_header.content_length
+
+            true
+        end
+=end
+=begin
+        def parse_header(header)
+            return false if @data.empty?
+
+            key = nil
+            if idx = @data.to_str.index("\r\n\r\n")
+                ctx = idx + 4
+                key = @data.to_str[0, ctx]
+                #key = Digest::MD5::digest(@data.to_str[0, ctx])
+                if obj = searchcache(key)
+                    @parser_nbytes = 0
+                    @data.read(ctx)
+                    @parser.reset
+                    @response_header = obj
+                    return true
+                end
+            end
+            begin
+                @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
+            rescue Rev::HttpClientParserError
+                on_error "invalid HTTP format, parsing fails"
+                @state = :invalid
+            end
+
+            return false unless @parser.finished?
+
+            if key
+                insertcache(key, header)
+            end
+
+            # Clear parsed data from the buffer
+            @data.read(@parser_nbytes)
+            @parser.reset
+            @parser_nbytes = 0
+
+            true
+        end
+=end
+        @@_cache = {}
+        @@_cache_keys = []
+        @@_cache_max = 200
+        def insertcache key, data
+            @@_cache[key] = data.dup
+            if @@_cache_keys.size > @@_cache_max
+                @@_cache.delete(@@_cache_keys.shift)
+            end
+        end
+        def searchcache key
+            if @@_cache.has_key?(key)
+                @@_cache_keys << key
+                @@_cache[key]
+            else
+                nil
+            end
+        end
+        def free
+            @content = ''
+            #@response_header = nil
+            #@chunk_header = nil
+            @response_time = nil
+            @_write_buffer.clear
+            #@_write_buffer = nil
+            @_read_watcher.detach if @_read_watcher.attached?
+            #@_read_watcher = nil
+            @_write_watcher.detach if @_write_watcher.attached?
+            #@_write_watcher = nil
+            @data.clear
+            #@data = nil
+            #@path = nil
+            #@method = nil
+            #@options = nil
+            #@starttime = nil
+            #@_io = nil
+            @writable_flag = nil
+            @send_flag = nil
+            @connected = false
+        end
+    end
+    class RevHttpClient < RevHttpConnection
+        def self::create host, port
+            #o = self::create_from_port(host, port)
+            o = self::new([host, port])
+            o.attach(Rev::Loop::default)
+            o.prepare(host, port)
+            o
+        end
+        attr_reader :result, :complete
+        attr_accessor :timeout, :useragent, :tfactory, :client_queue
+        def prepare host, port
+            init
+            @host = host
+            @port = port
+            @on_failure = nil
+            @on_success = nil
+            @client_queue = nil
+            _this = self
+            @queue = BaseHandler::get_handler('TaskQueue')
+            #@method_handle_response = method(:handle_response)
+            #以下のバグへの対応
+            #http://redmine.ruby-lang.org/issues/show/3786
+            @method_timeout = lambda{
+                _this.handle_timeout
+            }
+            #@method_timeout = method(:timeout)
+            #@method_call_on_success = method(:call_on_success)
+            #@method_call_on_failure = method(:call_on_failure)
+        end
+        def timeout= timeout
+            @timeout = timeout
+        end
+        def set_headers header
+            @headers = header
+        end
+        def set_header key, value
+            @headers[key] = value
+        end
+        def request_send methods, path, body = nil
+            @busy = true
+            #puts "req: #{object_id} / path: #{path}"
+            if body
+                options = {
+                    :head => @headers,
+                    :body => body
+                }
+                @headers['content-type'] = 'application/x-www-form-urlencoded'
+            else
+                options = {
+                    :head => @headers,
+                }
+            end
+            options[:cookies] = @cookie if @cookie
+            @timer = @tfactory.get
+            @timer.on_timer(&@method_timeout)
+            @exception = nil
+            begin
+                request(methods, path, options)
+            rescue
+                on_error("x")
+            end
+        end
+        def set_cookie cookie
+            @cookie = cookie
+        end
+        def handle_timeout
+            @exception = TimeoutError::new('timeout')
+            self.on_error
+        end
+        def set_callback on_success, on_failure
+            @on_success = on_success
+            @on_failure = on_failure
+        end
+        def has_callback
+            !@on_success.nil? and !@on_failure.nil?
+        end
+        def on_error(reason='error')
+            $REQUEST_COUNT += 1
+            super
+            @tfactory.collect(@timer) if @timer
+            @timer = nil
+            @busy = false
+            @response = @response_object
+            @response.time = @response_time
+            @on_failure.call(@exception, @response) if @on_failure
+            @exception = nil
+        end
+        def send_request_header
+            #無駄な処理を省くため上書き
+            head    = @options[:head]
+            body    = @options[:body]
+            cookies = @options[:cookies]
+
+            # Set the Content-Length if it hasn't been specified already and a body was given
+            # Default to Connection: close
+
+            # Build the request
+            request_header = HTTP_REQUEST_HEADER % [@method, @path]
+            request_header << encode_field('Content-Length', (body ? body.length : 0)) << encode_field('Connection', 'close')
+            for k, v in head
+                request_header << encode_field(k, v)
+            end
+            request_header << encode_cookies(cookies) if cookies
+            request_header << CRLF
+            write request_header
+        end
+        def on_request_complete
+            super
+            return if @exception
+            $REQUEST_COUNT += 1
+            @tfactory.collect(@timer)
+            @queue.add([@on_success, [self.to_response_object]])
+        end
+        def release
+            @client_queue.collect(self)
+        end
+    end
+end