internethakai 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. data/CHANGES +53 -0
  2. data/README +177 -0
  3. data/bin/hakaigen +10 -0
  4. data/bin/internethakai +9 -0
  5. data/internethakai.gemspec +27 -0
  6. data/lib/internethakai.rb +26 -0
  7. data/lib/internethakai.rb.~1~ +25 -0
  8. data/lib/internethakai/action.rb +403 -0
  9. data/lib/internethakai/client_handler.rb +175 -0
  10. data/lib/internethakai/client_queue.rb +27 -0
  11. data/lib/internethakai/concurrency_manager.rb +333 -0
  12. data/lib/internethakai/concurrency_manager.rb.~1~ +331 -0
  13. data/lib/internethakai/core.rb +146 -0
  14. data/lib/internethakai/executor.rb +129 -0
  15. data/lib/internethakai/generator.rb +32 -0
  16. data/lib/internethakai/hakairev.rb +119 -0
  17. data/lib/internethakai/hakairev/executor.rb +43 -0
  18. data/lib/internethakai/hakairev/http_client.rb +362 -0
  19. data/lib/internethakai/hakairev/http_client.rb.~1~ +371 -0
  20. data/lib/internethakai/hakairev/monkey.rb +70 -0
  21. data/lib/internethakai/hakairev/revpipe.rb +39 -0
  22. data/lib/internethakai/hakairev/task.rb +39 -0
  23. data/lib/internethakai/hakairev/time_register.rb +116 -0
  24. data/lib/internethakai/hakairev/timer_factory.rb +38 -0
  25. data/lib/internethakai/http_client.rb +120 -0
  26. data/lib/internethakai/logger.rb +52 -0
  27. data/lib/internethakai/main.rb +90 -0
  28. data/lib/internethakai/reporter.rb +143 -0
  29. data/lib/internethakai/response.rb +65 -0
  30. data/lib/internethakai/scenario.rb +98 -0
  31. data/lib/internethakai/scenario.tmpl +58 -0
  32. data/lib/internethakai/scenario_builder.rb +183 -0
  33. data/lib/internethakai/scenario_handler.rb +91 -0
  34. data/lib/internethakai/time_register.rb +53 -0
  35. data/lib/internethakai/util.rb +130 -0
  36. metadata +134 -0
@@ -0,0 +1,175 @@
1
+ module InternetHakai
2
+ # = httpクライアントをいじるクラス
3
+ class ClientHandler
4
+ UNIQUE_BY_THREAD = true
5
+ def self::on_config_load config
6
+ config['actions'] = config['actions'].map do |actconfig|
7
+ actconfig['usr_var'] = false
8
+ if(/%\(.*\)%/ =~ actconfig['path'])
9
+ actconfig['use_var'] = true
10
+ end
11
+ if(actconfig.has_key?('post_string') && /%25\(.*\)%25/ =~ actconfig['post_string'])
12
+ actconfig['use_var'] = true
13
+ end
14
+ actconfig
15
+ end
16
+ config
17
+ end
18
+ @@vars = Hash::new
19
+ attr_accessor :vars, :performance_id
20
+ def initialize scenario
21
+ @scenario = scenario
22
+ end
23
+ def send method, host, port, path
24
+ @client = get_client unless @client
25
+ end
26
+ def set_opt opt
27
+ @opt = opt
28
+ end
29
+ def set_var key, val
30
+ @scenario.vars[key] = val
31
+ end
32
+ def extend_vars str
33
+ str = str.to_s.clone
34
+ for k,v in @scenario.vars
35
+ str.gsub!("%(#{k})%", v)
36
+ end
37
+ str
38
+ end
39
+ def extend_vars2 str
40
+ str = str.to_s.clone
41
+ for k,v in @scenario.vars
42
+ str.gsub!("%25(#{k})%25", v)
43
+ end
44
+ str
45
+ end
46
+ def handle_url client, url, opt
47
+ opt['use_var'] ? extend_vars(url) : url
48
+ end
49
+ def handle_body client, body, opt
50
+ opt['use_var'] ? extend_vars2(body) : body
51
+ end
52
+ def handle_client client
53
+ @client = client
54
+ end
55
+ end
56
+
57
+ # = ケータイサイト用
58
+ class MobileClientHandler < ClientHandler
59
+ DOCOMO = 1
60
+ KDDI = 2
61
+ SOFTBANK = 3
62
+ def self::on_config_load config
63
+ if(config.has_key? "uid_path")
64
+ s = File::open(config["uid_path"]){|io|io.read}
65
+ lines = s.split("\n")
66
+ config["uids"] = lines.map{|i|i.split(",")}.sort_by{rand}
67
+ end
68
+ config
69
+ end
70
+ def handle_client client
71
+ super
72
+ uid = get_uid
73
+ case get_carrier
74
+ when DOCOMO
75
+ @client.set_header("http_x_dcmguid", uid)
76
+ when KDDI
77
+ @client.set_header("http_x_up_subno", uid)
78
+ when SOFTBANK
79
+ @client.set_header("http_x_jphone_uid", uid)
80
+ end
81
+ @client
82
+ end
83
+ def handle_url client, url, opt
84
+ url = super
85
+ q = Util::get_query(url)
86
+ if q["uid"] == "NULLGWDOCOMO"
87
+ url = Util::set_params(url.gsub("uid=NULLGWDOCOMO", ""), {"uid" => get_uid})
88
+ end
89
+ url
90
+ end
91
+ def get_uid
92
+ ids = @opts["uids"]
93
+ return ids unless ids.is_a? Array
94
+ idx = @performance_id % ids.size
95
+ ids[idx][1]
96
+ end
97
+ def get_carrier
98
+ ids = @opts["uids"]
99
+ idx = @performance_id % ids.size
100
+ data = ids[idx]
101
+ unless data
102
+ if @opt["carrier"]
103
+ @opt["carrier"]
104
+ end
105
+ DOCOMO
106
+ else
107
+ idx = @performance_id % ids.size
108
+ ids[idx][0]
109
+ end
110
+ end
111
+ end
112
+
113
+ # = ソーシャルアプリ用
114
+ class SocialClientHandler < ClientHandler
115
+ def self::on_config_load config
116
+ super
117
+ if(config.has_key? "opensocial_id_path")
118
+ s = File::open(config["opensocial_id_path"]){|io|io.read}
119
+ lines = s.split("\n")
120
+ config["opensocial_ids"] = lines.map{|i|i.split("\t")}.sort_by{rand}
121
+ config['opensocial_id_len'] = config['opensocial_ids'][0].size
122
+ elsif config['opensocial_ids']
123
+ r = []
124
+ config['opensocial_ids'].each do |i|
125
+ r << [i] unless i.is_a? Array
126
+ end
127
+ config['opensocial_ids'] = r
128
+ config['opensocial_id_len'] = config['opensocial_ids'][0].size
129
+ else
130
+ config['opensocial_ids'] = [1]
131
+ end
132
+ if !config['opensocial_app_id']
133
+ config['opensocial_app_id'] = 'app'
134
+ end
135
+ config['opensocial_app_id'] = config['opensocial_app_id'].to_s
136
+ config
137
+ end
138
+ def handle_client client
139
+ super
140
+ @client
141
+ end
142
+ def handle_body client, body, opt
143
+ body = super
144
+ if body[body.size-1] != 38
145
+ body += "&"
146
+ end
147
+ id = get_opensocial_id
148
+ app_id = @opt['opensocial_app_id']
149
+ body += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
150
+ end
151
+ def handle_url client, url, opt
152
+ url = super
153
+ if ! url.include? "?"
154
+ url += "?"
155
+ elsif url[url.size-1] == 63
156
+ url
157
+ elsif url[url.size-1] != 38
158
+ url += "&"
159
+ end
160
+ id = get_opensocial_id
161
+ #app_id = @opt['opensocial_app_id']
162
+ #url += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
163
+ url += 'opensocial_app_id=' + @opt['opensocial_app_id'] + '&opensocial_owner_id=' + id + '&opensocial_viewer_id=' + id
164
+ end
165
+ def get_opensocial_id
166
+ ids = @opt["opensocial_ids"]
167
+ id = ids[@performance_id % ids.size]
168
+ 1.upto(@opt['opensocial_id_len']-1) do |idx|
169
+ set_var(idx, id[idx])
170
+ end
171
+ r, = id
172
+ r.to_s
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,27 @@
1
+ module InternetHakai
2
+ class ClientQueue
3
+ def initialize size
4
+ @queue = []
5
+ @size = size
6
+ @ptr = size
7
+ end
8
+ def empty?
9
+ @ptr == 0
10
+ end
11
+ def get
12
+ if @ptr == 0
13
+ nil
14
+ else
15
+ @ptr -= 1
16
+ @queue.shift
17
+ end
18
+ end
19
+ def collect cl
20
+ @ptr += 1
21
+ @queue << cl
22
+ end
23
+ def add cl
24
+ @queue << cl
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,333 @@
1
+ module InternetHakai
2
+ class ConcurrencyManager
3
+ require 'thread'
4
+ def initialize
5
+ @fork_mode = false
6
+ @rev_mode = false
7
+ @process = 0
8
+ @suffix = ''
9
+ @mutex = nil
10
+ end
11
+ attr_accessor :suffix, :process
12
+ def prepare_rev
13
+ return if @rev_mode #2回実行しないように
14
+ require 'rubygems'
15
+ require 'rev'
16
+ require 'internethakai/hakairev'
17
+ #TimeRegistProcess::run
18
+ @config['http_client'] = 'RevHttpClient'
19
+ @config['scenario_executer'] = 'NonBlockScenarioExecuter'
20
+ @config['request_pool'] = 'RequestPoolNonThread'
21
+ @config['queue'] = 'TaskQueue'
22
+ #@config['response_handler'] = 'PipeTimeRegister'
23
+ @config['response_handler'] = 'RevTimeRegister'
24
+ @config['scan']
25
+ @rev_mode = true
26
+ end
27
+ def prepare_fork
28
+ return if @fork_mode
29
+ @concurrency = @config['max_scenario']
30
+ @process_max = @config['max_process'] || 2
31
+ @fork_mode = true
32
+ end
33
+ def run config, basepath, starttime
34
+ @config = config
35
+ @scenario_handler = BaseHandler::get_handler("ScenarioHandler")
36
+ @failure = false
37
+ @basepath = basepath
38
+ BaseHandler::set_config(@config)
39
+ @logger = BaseHandler::get_handler(@config["logger"])
40
+ @threads = {}
41
+
42
+ @concurrency = @config['max_scenario']
43
+ if @config['rev']
44
+ prepare_rev
45
+ end
46
+ if @config['max_process'] > 1
47
+ prepare_fork
48
+ end
49
+ @register = BaseHandler::get_handler(@config['response_handler'])
50
+ reporter = BaseHandler::get_handler(@config["reporter"])
51
+
52
+ digit = @config['max_request'].to_s.size
53
+ @id_tmpl = "%0#{digit}d"
54
+ if @config['max_process'] > 1
55
+ run_fork
56
+ else
57
+ run_process
58
+ end
59
+ end
60
+ def run_process
61
+ #並列数だけシナリオのコピーをつくっておく
62
+ @config['request_pool'] = 'RequestPoolNonThread' if @config["max_request"] <= 1
63
+ @scenarios = []
64
+ 1.upto(@concurrency) do
65
+ @scenarios << @scenario_handler.create_scenario(@suffix)
66
+ end
67
+ @scenario_handler.on_scenarios_created
68
+ $REQUEST_COUNT = 0
69
+ begin
70
+ if @rev_mode
71
+ rev_start
72
+ elsif @config["max_request"] <= 1
73
+ single_start
74
+ else
75
+ thread_start
76
+ end
77
+ rescue SystemExit
78
+ return
79
+ rescue Exception => e
80
+ raise e
81
+ puts "error: #{e.message}"
82
+ puts e.backtrace[0]
83
+ if @fork_mode
84
+ #死ぬ前に親に連絡
85
+ Process::kill(:USR1, Process::ppid)
86
+ else
87
+ report_collect
88
+ end
89
+ end
90
+ end
91
+ def single_start
92
+ th_cnt = 1
93
+ @threads[1] = Thread::current
94
+ @scenarios.each do |sc|
95
+ sc.init
96
+ end
97
+ #シングルスレッドのときは先にプールにためておく
98
+ run_executer th_cnt
99
+ if @failure
100
+ page_check
101
+ exit 1
102
+ else
103
+ exit 0
104
+ end
105
+ end
106
+ def page_check
107
+ keys = @result_record.keys
108
+ pages = []
109
+ keys.each do |key|
110
+ err = @result_record[key][:errorcount] || 0
111
+ if(err > 0)
112
+ pages << key
113
+ end
114
+ end
115
+ unless pages.empty?
116
+ @logger.run("ERROR URL: #{pages.join(',')}\n", 2)
117
+ end
118
+ end
119
+ def thread_start
120
+ @mutex = Mutex::new
121
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
122
+ @cv = ConditionVariable::new
123
+
124
+ @scenarios.each do |sc|
125
+ sc.init
126
+ end
127
+ Thread.abort_on_exception
128
+ @scenario_handler.set_on_turn_complete &method(:on_complete)
129
+ 1.upto(@config["max_request"]) do |th_cnt|
130
+ @threads[th_cnt] = Thread::new(th_cnt) do |th_cnt|
131
+ run_executer th_cnt
132
+ end
133
+ end
134
+ @threads[1].join
135
+ @threads[:main] = Thread::current
136
+ @mutex.synchronize{
137
+ @cv.wait(@mutex)
138
+ }
139
+ @threads.each do |k,th|
140
+ th.kill if th!=Thread::current && k!=:main
141
+ end
142
+ if @failure
143
+ exit 1
144
+ else
145
+ exit 0
146
+ end
147
+ end
148
+ def run_executer th_cnt
149
+ th_id = (sprintf(@id_tmpl, th_cnt) + @suffix).to_i
150
+ report = BaseHandler::get_handler(@config["response_handler"])
151
+ Thread.current[:result] = report
152
+ BaseHandler::set_thread_id(th_id)
153
+ cls = BaseHandler::get_class(@config['scenario_executer'])
154
+ @scenario_handler.set_on_turn_complete(&@register.method(:exec))
155
+ exec = cls::new
156
+ exec.set_on_complete &method(:on_complete)
157
+ @starttime = Time::now
158
+ exec.run
159
+ end
160
+ # = 1プロセス終了時に呼ばれる
161
+ def on_complete
162
+ total = Time::now - @starttime
163
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
164
+ @request_pool.close
165
+ #puts "thread collect start"
166
+ thread_collect
167
+ #puts "thread collect end"
168
+ report_collect
169
+ #puts "report collect end"
170
+ @logger.run("TotalTime: " + total.to_s + "\n", 2) unless @fork_mode
171
+ @failure = true if $error
172
+ @cv.signal if @cv
173
+ end
174
+ def create_id loop_c, th_c
175
+ (loop_c.to_s + sprintf(@id_tmpl, th_c) + @suffix).to_i
176
+ end
177
+ def thread_collect
178
+ count = 0
179
+ status = '.'
180
+ flag = false
181
+ count = 0
182
+ @threads.each do |k, th|
183
+ if(th[:result])
184
+ status = 't'
185
+ ect = 0
186
+ th[:result].get_record.each do |name,res|
187
+ ect += 1 if res[:errorcount] > 0
188
+ end
189
+ if ect > 0
190
+ status = 'e'
191
+ flag = true unless flag
192
+ end
193
+ #th.kill if th!=Thread::current && k!=:main
194
+ end
195
+ #@logger.run status, 2
196
+ end
197
+ @logger.run "\n", 2 unless @fork_mode
198
+ @logger.run "result: ", 2 unless @fork_mode
199
+ if flag
200
+ @logger.run "failure\n", 2 unless @fork_mode
201
+ else
202
+ @logger.run "success\n", 2 unless @fork_mode
203
+ end
204
+ @failure = flag
205
+ end
206
+ def report_collect
207
+ d = (Time::now - @starttime)*1000
208
+ if @fork_mode && @forkdb
209
+ @forkdb.transaction do
210
+ if not @forkdb['time']
211
+ @forkdb['time'] = d
212
+ elsif @forkdb['time'] < d
213
+ @forkdb['time'] = d
214
+ end
215
+ @forkdb['request_count'] = 0 unless @forkdb['request_count']
216
+ @forkdb['request_count'] += $REQUEST_COUNT
217
+ end
218
+ else
219
+ @logger.run "\nTimePerRequest: #{d / ($REQUEST_COUNT / @config['max_request'])}\n", 2
220
+ @logger.run "RequestPerSec: #{$REQUEST_COUNT / (d/1000)}\n", 2
221
+ end
222
+ record = ResponseRecord::new
223
+ @threads.each do |k, th|
224
+ if th[:result]
225
+ record += th[:result].get_record
226
+ end
227
+ end
228
+ @logger.run("\n\n", 2) unless @fork_mode
229
+ reporter = BaseHandler::get_handler(@config["reporter"])
230
+ reporter.run(record)
231
+ @result_record = record
232
+ end
233
+ def run_fork
234
+ raise "invalid config: fork: nil" unless @config['fork']
235
+ org_reporter = @config['reporter']
236
+
237
+ @logger.run("Process: #{@process_max}\n", 2)
238
+ @parent = false
239
+
240
+ flag = false
241
+ Signal::trap(:USR1){
242
+ return if flag
243
+ flag = true
244
+ #parent
245
+ puts "error: SIGUSR1"
246
+ @pids.each do |pid|
247
+ Process::kill(:USR2, pid)
248
+ end
249
+ }
250
+ Signal::trap(:INT){
251
+ flag = true
252
+ puts "error: SIGINT"
253
+ @pids.each do |pid|
254
+ Process::kill(:USR2, pid)
255
+ end
256
+ }
257
+ Signal::trap(:USR2){
258
+ #child
259
+ report_collect
260
+ exit
261
+ }
262
+
263
+ #各プロセスにはpstoreファイルに集計させる
264
+ @config['reporter'] = 'PStoreReporter'
265
+ fname = 'tmp.db'
266
+ fpath = File::join(@basepath, fname)
267
+ if FileTest::exists?(fpath)
268
+ raise 'cannot create db' unless FileTest::writable?(fpath)
269
+ raise 'cannot create db' unless File::delete(fpath)
270
+ end
271
+ db = PStore::new(fpath)
272
+ @forkdb = db
273
+
274
+ reporter = BaseHandler::get_handler(@config["reporter"])
275
+ reporter.set_dir(@basepath)
276
+ reporter.init_filename(fname)
277
+
278
+ begin
279
+ @pids = []
280
+ @process = @process_max
281
+ digit = @process_max.to_s.size
282
+ tmpl = "%0#{digit}d"
283
+ 1.upto(@process_max) do |i|
284
+ @pids << Process::fork {
285
+ @suffix = sprintf(tmpl, i)
286
+ run_process
287
+ }
288
+ #1プロセス目以外は出力を少なく
289
+ @scenario_handler.silence if i==2
290
+ end
291
+ @forkreport = false
292
+ @org_fork_reporter = org_reporter
293
+ results = Process::waitall
294
+ failure = false
295
+ results.each do |r|
296
+ if r[1].to_i != 0
297
+ failure = true
298
+ break
299
+ end
300
+ end
301
+ @logger.run "\n", 2
302
+ @logger.run "result: ", 2
303
+ unless failure
304
+ @logger.run "success\n", 2
305
+ else
306
+ @logger.run "failure\n", 2
307
+ end
308
+ fork_report
309
+ ensure
310
+ fork_report unless @forkreport
311
+ File::delete(fpath) if File::exists?(fpath)
312
+ end
313
+ end
314
+ def fork_report
315
+ @forkreport = true
316
+ require 'pstore'
317
+ result = nil
318
+ time = nil
319
+ count = nil
320
+ @forkdb.transaction do
321
+ result = @forkdb['result']
322
+ time = @forkdb['time']
323
+ count = @forkdb['request_count']
324
+ end
325
+ @logger.run "\nTimePerRequest: #{time / (count / @config['max_request_show'])}\n", 2 if time && count
326
+ @logger.run "RequestPerSec: #{count / (time/1000)}\n", 2 if time && count
327
+
328
+ reporter = BaseHandler::get_handler(@org_fork_reporter)
329
+ reporter.run(result)
330
+ reporter = nil
331
+ end
332
+ end
333
+ end