internethakai 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. data/CHANGES +53 -0
  2. data/README +177 -0
  3. data/bin/hakaigen +10 -0
  4. data/bin/internethakai +9 -0
  5. data/internethakai.gemspec +27 -0
  6. data/lib/internethakai.rb +26 -0
  7. data/lib/internethakai.rb.~1~ +25 -0
  8. data/lib/internethakai/action.rb +403 -0
  9. data/lib/internethakai/client_handler.rb +175 -0
  10. data/lib/internethakai/client_queue.rb +27 -0
  11. data/lib/internethakai/concurrency_manager.rb +333 -0
  12. data/lib/internethakai/concurrency_manager.rb.~1~ +331 -0
  13. data/lib/internethakai/core.rb +146 -0
  14. data/lib/internethakai/executor.rb +129 -0
  15. data/lib/internethakai/generator.rb +32 -0
  16. data/lib/internethakai/hakairev.rb +119 -0
  17. data/lib/internethakai/hakairev/executor.rb +43 -0
  18. data/lib/internethakai/hakairev/http_client.rb +362 -0
  19. data/lib/internethakai/hakairev/http_client.rb.~1~ +371 -0
  20. data/lib/internethakai/hakairev/monkey.rb +70 -0
  21. data/lib/internethakai/hakairev/revpipe.rb +39 -0
  22. data/lib/internethakai/hakairev/task.rb +39 -0
  23. data/lib/internethakai/hakairev/time_register.rb +116 -0
  24. data/lib/internethakai/hakairev/timer_factory.rb +38 -0
  25. data/lib/internethakai/http_client.rb +120 -0
  26. data/lib/internethakai/logger.rb +52 -0
  27. data/lib/internethakai/main.rb +90 -0
  28. data/lib/internethakai/reporter.rb +143 -0
  29. data/lib/internethakai/response.rb +65 -0
  30. data/lib/internethakai/scenario.rb +98 -0
  31. data/lib/internethakai/scenario.tmpl +58 -0
  32. data/lib/internethakai/scenario_builder.rb +183 -0
  33. data/lib/internethakai/scenario_handler.rb +91 -0
  34. data/lib/internethakai/time_register.rb +53 -0
  35. data/lib/internethakai/util.rb +130 -0
  36. metadata +134 -0
@@ -0,0 +1,129 @@
1
+ module InternetHakai
2
+ class RequestPool < BaseHandler
3
+ UNIQUE_BY_THREAD = false
4
+ require 'thread'
5
+ def initialize id
6
+ @queue = Queue::new
7
+ @enable = true
8
+ @closed = false
9
+ end
10
+ def clear
11
+ @queue.clear
12
+ @queue = Queue::new
13
+ end
14
+ def add action
15
+ return unless @enable
16
+ @queue.enq(action)
17
+ end
18
+ def empty?
19
+ @queue.empty?
20
+ end
21
+ def get
22
+ return if @closed
23
+ r = @queue.deq
24
+ if ! @enable and @queue.empty?
25
+ #enable=false で キューが空になった時点でclose
26
+ close
27
+ end
28
+ return r
29
+ end
30
+ def close
31
+ @closed = true
32
+ 1.upto(@queue.num_waiting) do
33
+ @queue.enq(-1)
34
+ end
35
+ @queue.clear
36
+ end
37
+ def quit
38
+ @enable = false
39
+ #enable=false で キューが空になった時点でclose
40
+ close if @queue.empty?
41
+ end
42
+ end
43
+ class SimpleQueue < BaseHandler
44
+ UNIQUE_BY_THREAD = false
45
+ def add *ar
46
+ task, arg = ar
47
+ end
48
+ end
49
+ class RequestPoolNonThread < BaseHandler
50
+ UNIQUE_BY_THREAD = false
51
+ def initialize id
52
+ @queue = []
53
+ @closed = false
54
+ @enable = true
55
+ end
56
+ def clear
57
+ @queue = []
58
+ end
59
+ def empty?
60
+ @queue.empty?
61
+ end
62
+ def add action
63
+ return unless @enable
64
+ @queue << action
65
+ end
66
+ def get
67
+ return -1 if @closed
68
+ if ! @enable and @queue.empty?
69
+ close
70
+ return -1
71
+ end
72
+ @queue.shift
73
+ end
74
+ def close
75
+ @closed = true
76
+ end
77
+ def quit
78
+ @enable = false
79
+ close if @queue.empty?
80
+ end
81
+ end
82
+ # = シナリオを実行するクラス
83
+ class ScenarioExecuter
84
+ def initialize
85
+ @scenario_handler = BaseHandler::get_handler("ScenarioHandler")
86
+ @enable = true
87
+ @logger = BaseHandler::get_handler('Logger')
88
+ @config = BaseHandler::get_config
89
+ @max_request = @config['max_request']
90
+ @register = BaseHandler::get_handler(@config['response_handler'])
91
+ client_cls = BaseHandler::get_class(@config['http_client'])
92
+ cl = client_cls::create(@config['host'], @config['port'])
93
+ cl.timeout = @config['timeout']
94
+ cl.set_headers(@config['header'])
95
+ @client = cl
96
+ end
97
+ # すべてのシナリオの終了時に実行されます
98
+ def set_on_complete &block
99
+ @on_complete_callback = block
100
+ end
101
+ def on_complete
102
+ return if @scenario_handler.closed?
103
+ if @on_complete_callback
104
+ @scenario_handler.close
105
+ @on_complete_callback.call
106
+ end
107
+ end
108
+ def run
109
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
110
+ pool = @request_pool
111
+ client = @client
112
+ scenario_handler = @scenario_handler
113
+ register = @register
114
+ count = 0
115
+ while true
116
+ break unless @enable
117
+ if scenario_handler.complete?
118
+ on_complete
119
+ end
120
+ act = pool.get
121
+ break if act.nil? or act==-1
122
+ act.set_client(client)
123
+ act.run
124
+ register.exec
125
+ end
126
+ on_complete
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,32 @@
1
+ module InternetHakai
2
+ class Generator
3
+ require 'erb'
4
+ @@readline = require 'readline'
5
+ def build
6
+ s = open(File::join(File::dirname(__FILE__), 'scenario.tmpl')){|io|io.read}
7
+ template = ERB::new(s)
8
+ puts "your site\n ex. http://example.com"
9
+ if @@readline
10
+ domain = Readline.readline("> ", true)
11
+ else
12
+ domain = gets
13
+ end
14
+ if(domain)
15
+ domain.chomp!
16
+ domain = 'http://' + domain unless /^https?:\/\// =~ domain
17
+ else
18
+ domain = 'http://example.com'
19
+ end
20
+ social = false
21
+ data = template.result(binding)
22
+ fname = 'scenario'
23
+ idx = 0
24
+ while(File::exists?(fname+"-#{idx}.yml"))
25
+ idx += 1
26
+ end
27
+ fname = fname + "-#{idx}.yml"
28
+ open(fname, 'w'){|io|io.write(data)}
29
+ puts "save to #{fname}"
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,119 @@
1
+ require 'internethakai/hakairev/executor'
2
+ require 'internethakai/hakairev/http_client'
3
+ require 'internethakai/hakairev/task'
4
+ require 'internethakai/hakairev/time_register'
5
+ require 'internethakai/hakairev/timer_factory'
6
+ require 'internethakai/hakairev/revpipe'
7
+ require 'internethakai/hakairev/monkey'
8
+
9
+ module InternetHakai
10
+ class ConcurrencyManager
11
+ def rev_executer
12
+ @register.exec
13
+ @scenario_executer.run_one_action
14
+ end
15
+ def rev_on_complete
16
+ @queue.clear
17
+ on_complete
18
+ rev_stop
19
+ end
20
+ def rev_scenario_push
21
+ flag = true
22
+ @scenarios.each do |sc|
23
+ flag = sc.next
24
+ end
25
+ unless flag
26
+ #@logger.run("all scenario push\n", 2)
27
+ @request_pool.quit
28
+ @threads[:scenario].kill if @threads[:scenario]
29
+ end
30
+ flag
31
+ end
32
+ def start_task_thread
33
+ q = @queue
34
+ Thread::start(q) do |q|
35
+ while @rev_running
36
+ q.run
37
+ end
38
+ end
39
+ end
40
+ def rev_start
41
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
42
+ th_cnt = 1
43
+ th_id = (sprintf(@id_tmpl, th_cnt) + @suffix).to_i
44
+ BaseHandler::set_thread_id(th_id)
45
+ @queue = BaseHandler::get_handler('TaskQueue')
46
+ TimerFactory::new(@config['timeout'], @config['max_request'], Rev::Loop::default)
47
+ @client_queue = ClientQueue::new(@config['max_request'])
48
+ 1.upto(@config['max_request']) do |i|
49
+ client_cls = BaseHandler::get_class(@config['http_client'])
50
+ cl = client_cls::create(@config['host'], @config['port'])
51
+ cl.timeout = @config['timeout']
52
+ cl.set_headers(@config['header'])
53
+ cl.tfactory = TimerFactory::instance
54
+ cl.client_queue = @client_queue
55
+ @client_queue.add(cl)
56
+ end
57
+
58
+ Thread::current[:heartbeat] = Time::now
59
+ report = BaseHandler::get_handler(@config["response_handler"])
60
+ Thread.current[:result] = report
61
+
62
+ @scenarios.each do |sc|
63
+ #最初のアクションだけセット
64
+ #sc.each do |act|
65
+ # act.client_queue = @client_queue
66
+ #end
67
+ sc.init
68
+ end
69
+
70
+ @threads[:main] = Thread::current
71
+ @starttime = Time::now
72
+ @scenario_handler.set_on_complete(&method(:rev_on_complete))
73
+ @scenario_handler.set_on_turn_complete(&@register.method(:exec))
74
+ #rev_executer_run
75
+ @rev_running = true
76
+ #start_task_thread
77
+ rev_loop_run
78
+ if @failure
79
+ exit 1
80
+ else
81
+ exit 0
82
+ end
83
+ end
84
+ def rev_loop_run
85
+ loop = Rev::Loop::default
86
+ q = @queue
87
+ rq = @request_pool
88
+ clq = @client_queue
89
+ while @rev_running
90
+ flag = false
91
+ if loop.has_active_watchers?
92
+ loop.run_nonblock
93
+ end
94
+ if !rq.empty? and !clq.empty?
95
+ a = rq.get
96
+ cl = clq.get
97
+ #a.client_queue = clq
98
+ #puts "set: #{cl.object_id}"
99
+ a.set_client(cl)
100
+ a.run
101
+ flag = true
102
+ end
103
+ if loop.has_active_watchers?
104
+ loop.run_nonblock
105
+ end
106
+ if !q.empty?
107
+ q.run
108
+ flag = true
109
+ end
110
+ if !flag
111
+ loop.run_once
112
+ end
113
+ end
114
+ end
115
+ def rev_stop
116
+ @rev_running = false
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,43 @@
1
+ module InternetHakai
2
+ class NonBlockScenarioExecuter < ScenarioExecuter
3
+ def initialize
4
+ super
5
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
6
+ @concurrency = @config['concurrency']
7
+ @max_request = @config['max_request']
8
+ @request_counter = @max_request
9
+ @acts = []
10
+ @timer = nil
11
+ @queue = BaseHandler::get_handler('TaskQueue')
12
+ @method_dispatch_next = method(:dispatch_next)
13
+ @method_run_one = method(:run_one)
14
+ end
15
+ attr_accessor :timer
16
+ #max_request分のactionが終了
17
+ def on_action_complete &block
18
+ if block
19
+ @on_action_complete_callback = block
20
+ elsif @on_action_complete_callback
21
+ @on_action_complete_callback.call
22
+ end
23
+ end
24
+ def dispatch_next
25
+ @queue.add(@method_run_one)
26
+ end
27
+ def run_one
28
+ act = @request_pool.get
29
+ #puts "act: #{act.class} :#{act.object_id}" if act!=-1 && act
30
+ if act.nil?
31
+ dispatch_next
32
+ return
33
+ elsif act == -1
34
+ #-1が返ったらおわり
35
+ #dispatch_next
36
+ return
37
+ end
38
+ #actionのcallbackに次の実行を設定
39
+ act.set_on_complete(&@method_dispatch_next) unless act.has_on_complete?
40
+ act.run
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,362 @@
1
+ module InternetHakai
2
+ class RevHttpConnection < Rev::HttpClient
3
+ def initialize args
4
+ @remote_addr, @remote_port = args
5
+ soc = TCPConnectSocket::new(::Socket::AF_INET, @remote_addr, @remote_port)
6
+ super(soc)
7
+ end
8
+ def init
9
+ @content = ''
10
+ @writable_flag = false
11
+ @send_flag = false
12
+ @busy = false
13
+ @on_request_complete = nil
14
+ @requested = false
15
+ @response_object = InternetHakai::ResponseObject::new
16
+ self
17
+ end
18
+ attr_reader :connected, :busy
19
+ def on_read s
20
+ super
21
+ end
22
+ def on_readable
23
+ begin
24
+ on_read @_io.read_nonblock(INPUT_SIZE)
25
+ rescue Errno::EAGAIN
26
+ rescue Errno::ECONNRESET, EOFError
27
+ rescue IOError => e
28
+ @exception = e
29
+ on_error('io error')
30
+ rescue Exception => e
31
+ @exception = e
32
+ on_error('io error')
33
+ end
34
+ end
35
+ def reconnect
36
+ @requested = false
37
+ @busy = false
38
+ free
39
+ socket = TCPConnectSocket::new(::Socket::AF_INET, @remote_addr, @remote_port)
40
+
41
+ @_io = socket
42
+ #@_write_buffer ||= ::IO::Buffer.new
43
+ @_read_watcher = Watcher.new(socket, self, :r)
44
+ @_write_watcher = Watcher.new(socket, self, :w)
45
+ @address_family, @remote_port, @remote_host, @remote_addr = socket.peeraddr
46
+
47
+
48
+ @state = :response_header
49
+ #@data = ::IO::Buffer.new
50
+
51
+ @response_header = Rev::HttpResponseHeader.new
52
+ @chunk_header = Rev::HttpChunkHeader.new
53
+ #initialize(socket)
54
+
55
+ @_connector = Connector::new(self, socket)
56
+ @_connector.attach(@loop)
57
+ end
58
+ def close
59
+ super
60
+ end
61
+ def on_writable
62
+ @writable_flag = true unless @writable_flag
63
+ # writable + send 両方そろったら開始時間
64
+ if @send_flag && !@starttime
65
+ @starttime = Time::now
66
+ end
67
+ begin
68
+ @_write_buffer.write_to(@_io)
69
+
70
+ rescue Errno::EPIPE, Errno::ECONNRESET => e
71
+ rescue IOError => e
72
+ rescue Exception => e
73
+ @exception = e
74
+ on_error('io error')
75
+ end
76
+ if @_write_buffer.empty?
77
+ disable_write_watcher
78
+ on_write_complete
79
+ end
80
+ end
81
+ def send_request
82
+ @send_flag = true
83
+ @starttime = Time::now
84
+ super
85
+ end
86
+ def attached?
87
+ if @_connector
88
+ @_connector.attached?
89
+ elsif @_read_watcher
90
+ @_read_watcher.attached?
91
+ elsif @_resolver
92
+ @_resolver.attached
93
+ else
94
+ super
95
+ end
96
+ end
97
+ def on_body_data(data)
98
+ @content += data
99
+ end
100
+ def on_request_complete
101
+ @response_time = Time::now - @starttime
102
+ self.close
103
+ end
104
+ def on_connect_failed
105
+ @exception = SocketError::new('connection error')
106
+ on_error('')
107
+ end
108
+ def on_resolve_failed
109
+ @exception = SocketError::new('dns error')
110
+ on_error('')
111
+ end
112
+ def on_failure
113
+ @exception = SocketError::new
114
+ on_error('')
115
+ end
116
+ def on_error(reason='error')
117
+ unless @starttime.nil?
118
+ @response_time = Time::now - @starttime
119
+ else
120
+ @response_time = 0
121
+ end
122
+ close if @_io and !@_io.closed?
123
+ @requested = false
124
+ @exception = reason if @exception.nil?
125
+ @busy = false
126
+ end
127
+ def to_response_object
128
+ r = @response_object
129
+ r.body = @content.to_s
130
+ r.status = @response_header.http_status.to_i
131
+ r.content_type = @response_header["CONTENT_TYPE"].to_s
132
+ r.location = @response_header["LOCATION"]
133
+ r.cookie = @response_header['SET_COOKIE']
134
+ r.time = @response_time
135
+ r
136
+ end
137
+ def attach loop
138
+ @loop = loop unless @loop
139
+ if @_connector
140
+ @_connector.attach loop
141
+ elsif @_read_watcher
142
+ @_read_watcher.attach loop
143
+ elsif @_resolver
144
+ @_resolver.attache loop
145
+ else
146
+ return
147
+ end
148
+ end
149
+ =begin
150
+ def parse_response_header
151
+ return false unless parse_header(@response_header)
152
+
153
+ unless @response_header.http_status and @response_header.http_reason
154
+ on_error "no HTTP response"
155
+ @state = :invalid
156
+ return false
157
+ end
158
+
159
+ on_response_header(@response_header)
160
+
161
+ @state = :body
162
+ @bytes_remaining = @response_header.content_length
163
+
164
+ true
165
+ end
166
+ =end
167
+ =begin
168
+ def parse_header(header)
169
+ return false if @data.empty?
170
+
171
+ key = nil
172
+ if idx = @data.to_str.index("\r\n\r\n")
173
+ ctx = idx + 4
174
+ key = @data.to_str[0, ctx]
175
+ #key = Digest::MD5::digest(@data.to_str[0, ctx])
176
+ if obj = searchcache(key)
177
+ @parser_nbytes = 0
178
+ @data.read(ctx)
179
+ @parser.reset
180
+ @response_header = obj
181
+ return true
182
+ end
183
+ end
184
+ begin
185
+ @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
186
+ rescue Rev::HttpClientParserError
187
+ on_error "invalid HTTP format, parsing fails"
188
+ @state = :invalid
189
+ end
190
+
191
+ return false unless @parser.finished?
192
+
193
+ if key
194
+ insertcache(key, header)
195
+ end
196
+
197
+ # Clear parsed data from the buffer
198
+ @data.read(@parser_nbytes)
199
+ @parser.reset
200
+ @parser_nbytes = 0
201
+
202
+ true
203
+ end
204
+ =end
205
+ @@_cache = {}
206
+ @@_cache_keys = []
207
+ @@_cache_max = 200
208
+ def insertcache key, data
209
+ @@_cache[key] = data.dup
210
+ if @@_cache_keys.size > @@_cache_max
211
+ @@_cache.delete(@@_cache_keys.shift)
212
+ end
213
+ end
214
+ def searchcache key
215
+ if @@_cache.has_key?(key)
216
+ @@_cache_keys << key
217
+ @@_cache[key]
218
+ else
219
+ nil
220
+ end
221
+ end
222
+ def free
223
+ @content = ''
224
+ #@response_header = nil
225
+ #@chunk_header = nil
226
+ @response_time = nil
227
+ @_write_buffer.clear
228
+ #@_write_buffer = nil
229
+ @_read_watcher.detach if @_read_watcher.attached?
230
+ #@_read_watcher = nil
231
+ @_write_watcher.detach if @_write_watcher.attached?
232
+ #@_write_watcher = nil
233
+ @data.clear
234
+ #@data = nil
235
+ #@path = nil
236
+ #@method = nil
237
+ #@options = nil
238
+ #@starttime = nil
239
+ #@_io = nil
240
+ @writable_flag = nil
241
+ @send_flag = nil
242
+ @connected = false
243
+ end
244
+ end
245
+ class RevHttpClient < RevHttpConnection
246
+ def self::create host, port
247
+ #o = self::create_from_port(host, port)
248
+ o = self::new([host, port])
249
+ o.attach(Rev::Loop::default)
250
+ o.prepare(host, port)
251
+ o
252
+ end
253
+ attr_reader :result, :complete
254
+ attr_accessor :timeout, :useragent, :tfactory, :client_queue
255
+ def prepare host, port
256
+ init
257
+ @host = host
258
+ @port = port
259
+ @on_failure = nil
260
+ @on_success = nil
261
+ @client_queue = nil
262
+ _this = self
263
+ @queue = BaseHandler::get_handler('TaskQueue')
264
+ #@method_handle_response = method(:handle_response)
265
+ #以下のバグへの対応
266
+ #http://redmine.ruby-lang.org/issues/show/3786
267
+ @method_timeout = lambda{
268
+ _this.handle_timeout
269
+ }
270
+ #@method_timeout = method(:timeout)
271
+ #@method_call_on_success = method(:call_on_success)
272
+ #@method_call_on_failure = method(:call_on_failure)
273
+ end
274
+ def timeout= timeout
275
+ @timeout = timeout
276
+ end
277
+ def set_headers header
278
+ @headers = header
279
+ end
280
+ def set_header key, value
281
+ @headers[key] = value
282
+ end
283
+ def request_send methods, path, body = nil
284
+ @busy = true
285
+ #puts "req: #{object_id} / path: #{path}"
286
+ if body
287
+ options = {
288
+ :head => @headers,
289
+ :body => body
290
+ }
291
+ @headers['content-type'] = 'application/x-www-form-urlencoded'
292
+ else
293
+ options = {
294
+ :head => @headers,
295
+ }
296
+ end
297
+ options[:cookies] = @cookie if @cookie
298
+ @timer = @tfactory.get
299
+ @timer.on_timer(&@method_timeout)
300
+ @exception = nil
301
+ begin
302
+ request(methods, path, options)
303
+ rescue
304
+ on_error("x")
305
+ end
306
+ end
307
+ def set_cookie cookie
308
+ @cookie = cookie
309
+ end
310
+ def handle_timeout
311
+ @exception = TimeoutError::new('timeout')
312
+ self.on_error
313
+ end
314
+ def set_callback on_success, on_failure
315
+ @on_success = on_success
316
+ @on_failure = on_failure
317
+ end
318
+ def has_callback
319
+ !@on_success.nil? and !@on_failure.nil?
320
+ end
321
+ def on_error(reason='error')
322
+ $REQUEST_COUNT += 1
323
+ super
324
+ @tfactory.collect(@timer) if @timer
325
+ @timer = nil
326
+ @busy = false
327
+ @response = @response_object
328
+ @response.time = @response_time
329
+ @on_failure.call(@exception, @response) if @on_failure
330
+ @exception = nil
331
+ end
332
+ def send_request_header
333
+ #無駄な処理を省くため上書き
334
+ head = @options[:head]
335
+ body = @options[:body]
336
+ cookies = @options[:cookies]
337
+
338
+ # Set the Content-Length if it hasn't been specified already and a body was given
339
+ # Default to Connection: close
340
+
341
+ # Build the request
342
+ request_header = HTTP_REQUEST_HEADER % [@method, @path]
343
+ request_header << encode_field('Content-Length', (body ? body.length : 0)) << encode_field('Connection', 'close')
344
+ for k, v in head
345
+ request_header << encode_field(k, v)
346
+ end
347
+ request_header << encode_cookies(cookies) if cookies
348
+ request_header << CRLF
349
+ write request_header
350
+ end
351
+ def on_request_complete
352
+ super
353
+ return if @exception
354
+ $REQUEST_COUNT += 1
355
+ @tfactory.collect(@timer)
356
+ @queue.add([@on_success, [self.to_response_object]])
357
+ end
358
+ def release
359
+ @client_queue.collect(self)
360
+ end
361
+ end
362
+ end