internethakai 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. data/CHANGES +53 -0
  2. data/README +177 -0
  3. data/bin/hakaigen +10 -0
  4. data/bin/internethakai +9 -0
  5. data/internethakai.gemspec +27 -0
  6. data/lib/internethakai.rb +26 -0
  7. data/lib/internethakai.rb.~1~ +25 -0
  8. data/lib/internethakai/action.rb +403 -0
  9. data/lib/internethakai/client_handler.rb +175 -0
  10. data/lib/internethakai/client_queue.rb +27 -0
  11. data/lib/internethakai/concurrency_manager.rb +333 -0
  12. data/lib/internethakai/concurrency_manager.rb.~1~ +331 -0
  13. data/lib/internethakai/core.rb +146 -0
  14. data/lib/internethakai/executor.rb +129 -0
  15. data/lib/internethakai/generator.rb +32 -0
  16. data/lib/internethakai/hakairev.rb +119 -0
  17. data/lib/internethakai/hakairev/executor.rb +43 -0
  18. data/lib/internethakai/hakairev/http_client.rb +362 -0
  19. data/lib/internethakai/hakairev/http_client.rb.~1~ +371 -0
  20. data/lib/internethakai/hakairev/monkey.rb +70 -0
  21. data/lib/internethakai/hakairev/revpipe.rb +39 -0
  22. data/lib/internethakai/hakairev/task.rb +39 -0
  23. data/lib/internethakai/hakairev/time_register.rb +116 -0
  24. data/lib/internethakai/hakairev/timer_factory.rb +38 -0
  25. data/lib/internethakai/http_client.rb +120 -0
  26. data/lib/internethakai/logger.rb +52 -0
  27. data/lib/internethakai/main.rb +90 -0
  28. data/lib/internethakai/reporter.rb +143 -0
  29. data/lib/internethakai/response.rb +65 -0
  30. data/lib/internethakai/scenario.rb +98 -0
  31. data/lib/internethakai/scenario.tmpl +58 -0
  32. data/lib/internethakai/scenario_builder.rb +183 -0
  33. data/lib/internethakai/scenario_handler.rb +91 -0
  34. data/lib/internethakai/time_register.rb +53 -0
  35. data/lib/internethakai/util.rb +130 -0
  36. metadata +134 -0
@@ -0,0 +1,175 @@
1
+ module InternetHakai
2
+ # = httpクライアントをいじるクラス
3
+ class ClientHandler
4
+ UNIQUE_BY_THREAD = true
5
+ def self::on_config_load config
6
+ config['actions'] = config['actions'].map do |actconfig|
7
+ actconfig['usr_var'] = false
8
+ if(/%\(.*\)%/ =~ actconfig['path'])
9
+ actconfig['use_var'] = true
10
+ end
11
+ if(actconfig.has_key?('post_string') && /%25\(.*\)%25/ =~ actconfig['post_string'])
12
+ actconfig['use_var'] = true
13
+ end
14
+ actconfig
15
+ end
16
+ config
17
+ end
18
+ @@vars = Hash::new
19
+ attr_accessor :vars, :performance_id
20
+ def initialize scenario
21
+ @scenario = scenario
22
+ end
23
+ def send method, host, port, path
24
+ @client = get_client unless @client
25
+ end
26
+ def set_opt opt
27
+ @opt = opt
28
+ end
29
+ def set_var key, val
30
+ @scenario.vars[key] = val
31
+ end
32
+ def extend_vars str
33
+ str = str.to_s.clone
34
+ for k,v in @scenario.vars
35
+ str.gsub!("%(#{k})%", v)
36
+ end
37
+ str
38
+ end
39
+ def extend_vars2 str
40
+ str = str.to_s.clone
41
+ for k,v in @scenario.vars
42
+ str.gsub!("%25(#{k})%25", v)
43
+ end
44
+ str
45
+ end
46
+ def handle_url client, url, opt
47
+ opt['use_var'] ? extend_vars(url) : url
48
+ end
49
+ def handle_body client, body, opt
50
+ opt['use_var'] ? extend_vars2(body) : body
51
+ end
52
+ def handle_client client
53
+ @client = client
54
+ end
55
+ end
56
+
57
+ # = ケータイサイト用
58
+ class MobileClientHandler < ClientHandler
59
+ DOCOMO = 1
60
+ KDDI = 2
61
+ SOFTBANK = 3
62
+ def self::on_config_load config
63
+ if(config.has_key? "uid_path")
64
+ s = File::open(config["uid_path"]){|io|io.read}
65
+ lines = s.split("\n")
66
+ config["uids"] = lines.map{|i|i.split(",")}.sort_by{rand}
67
+ end
68
+ config
69
+ end
70
+ def handle_client client
71
+ super
72
+ uid = get_uid
73
+ case get_carrier
74
+ when DOCOMO
75
+ @client.set_header("http_x_dcmguid", uid)
76
+ when KDDI
77
+ @client.set_header("http_x_up_subno", uid)
78
+ when SOFTBANK
79
+ @client.set_header("http_x_jphone_uid", uid)
80
+ end
81
+ @client
82
+ end
83
+ def handle_url client, url, opt
84
+ url = super
85
+ q = Util::get_query(url)
86
+ if q["uid"] == "NULLGWDOCOMO"
87
+ url = Util::set_params(url.gsub("uid=NULLGWDOCOMO", ""), {"uid" => get_uid})
88
+ end
89
+ url
90
+ end
91
+ def get_uid
92
+ ids = @opts["uids"]
93
+ return ids unless ids.is_a? Array
94
+ idx = @performance_id % ids.size
95
+ ids[idx][1]
96
+ end
97
+ def get_carrier
98
+ ids = @opts["uids"]
99
+ idx = @performance_id % ids.size
100
+ data = ids[idx]
101
+ unless data
102
+ if @opt["carrier"]
103
+ @opt["carrier"]
104
+ end
105
+ DOCOMO
106
+ else
107
+ idx = @performance_id % ids.size
108
+ ids[idx][0]
109
+ end
110
+ end
111
+ end
112
+
113
+ # = ソーシャルアプリ用
114
+ class SocialClientHandler < ClientHandler
115
+ def self::on_config_load config
116
+ super
117
+ if(config.has_key? "opensocial_id_path")
118
+ s = File::open(config["opensocial_id_path"]){|io|io.read}
119
+ lines = s.split("\n")
120
+ config["opensocial_ids"] = lines.map{|i|i.split("\t")}.sort_by{rand}
121
+ config['opensocial_id_len'] = config['opensocial_ids'][0].size
122
+ elsif config['opensocial_ids']
123
+ r = []
124
+ config['opensocial_ids'].each do |i|
125
+ r << [i] unless i.is_a? Array
126
+ end
127
+ config['opensocial_ids'] = r
128
+ config['opensocial_id_len'] = config['opensocial_ids'][0].size
129
+ else
130
+ config['opensocial_ids'] = [1]
131
+ end
132
+ if !config['opensocial_app_id']
133
+ config['opensocial_app_id'] = 'app'
134
+ end
135
+ config['opensocial_app_id'] = config['opensocial_app_id'].to_s
136
+ config
137
+ end
138
+ def handle_client client
139
+ super
140
+ @client
141
+ end
142
+ def handle_body client, body, opt
143
+ body = super
144
+ if body[body.size-1] != 38
145
+ body += "&"
146
+ end
147
+ id = get_opensocial_id
148
+ app_id = @opt['opensocial_app_id']
149
+ body += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
150
+ end
151
+ def handle_url client, url, opt
152
+ url = super
153
+ if ! url.include? "?"
154
+ url += "?"
155
+ elsif url[url.size-1] == 63
156
+ url
157
+ elsif url[url.size-1] != 38
158
+ url += "&"
159
+ end
160
+ id = get_opensocial_id
161
+ #app_id = @opt['opensocial_app_id']
162
+ #url += "opensocial_app_id=#{app_id}&opensocial_owner_id=#{id}&opensocial_viewer_id=#{id}"
163
+ url += 'opensocial_app_id=' + @opt['opensocial_app_id'] + '&opensocial_owner_id=' + id + '&opensocial_viewer_id=' + id
164
+ end
165
+ def get_opensocial_id
166
+ ids = @opt["opensocial_ids"]
167
+ id = ids[@performance_id % ids.size]
168
+ 1.upto(@opt['opensocial_id_len']-1) do |idx|
169
+ set_var(idx, id[idx])
170
+ end
171
+ r, = id
172
+ r.to_s
173
+ end
174
+ end
175
+ end
@@ -0,0 +1,27 @@
1
+ module InternetHakai
2
+ class ClientQueue
3
+ def initialize size
4
+ @queue = []
5
+ @size = size
6
+ @ptr = size
7
+ end
8
+ def empty?
9
+ @ptr == 0
10
+ end
11
+ def get
12
+ if @ptr == 0
13
+ nil
14
+ else
15
+ @ptr -= 1
16
+ @queue.shift
17
+ end
18
+ end
19
+ def collect cl
20
+ @ptr += 1
21
+ @queue << cl
22
+ end
23
+ def add cl
24
+ @queue << cl
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,333 @@
1
+ module InternetHakai
2
+ class ConcurrencyManager
3
+ require 'thread'
4
+ def initialize
5
+ @fork_mode = false
6
+ @rev_mode = false
7
+ @process = 0
8
+ @suffix = ''
9
+ @mutex = nil
10
+ end
11
+ attr_accessor :suffix, :process
12
+ def prepare_rev
13
+ return if @rev_mode #2回実行しないように
14
+ require 'rubygems'
15
+ require 'rev'
16
+ require 'internethakai/hakairev'
17
+ #TimeRegistProcess::run
18
+ @config['http_client'] = 'RevHttpClient'
19
+ @config['scenario_executer'] = 'NonBlockScenarioExecuter'
20
+ @config['request_pool'] = 'RequestPoolNonThread'
21
+ @config['queue'] = 'TaskQueue'
22
+ #@config['response_handler'] = 'PipeTimeRegister'
23
+ @config['response_handler'] = 'RevTimeRegister'
24
+ @config['scan']
25
+ @rev_mode = true
26
+ end
27
+ def prepare_fork
28
+ return if @fork_mode
29
+ @concurrency = @config['max_scenario']
30
+ @process_max = @config['max_process'] || 2
31
+ @fork_mode = true
32
+ end
33
+ def run config, basepath, starttime
34
+ @config = config
35
+ @scenario_handler = BaseHandler::get_handler("ScenarioHandler")
36
+ @failure = false
37
+ @basepath = basepath
38
+ BaseHandler::set_config(@config)
39
+ @logger = BaseHandler::get_handler(@config["logger"])
40
+ @threads = {}
41
+
42
+ @concurrency = @config['max_scenario']
43
+ if @config['rev']
44
+ prepare_rev
45
+ end
46
+ if @config['max_process'] > 1
47
+ prepare_fork
48
+ end
49
+ @register = BaseHandler::get_handler(@config['response_handler'])
50
+ reporter = BaseHandler::get_handler(@config["reporter"])
51
+
52
+ digit = @config['max_request'].to_s.size
53
+ @id_tmpl = "%0#{digit}d"
54
+ if @config['max_process'] > 1
55
+ run_fork
56
+ else
57
+ run_process
58
+ end
59
+ end
60
+ def run_process
61
+ #並列数だけシナリオのコピーをつくっておく
62
+ @config['request_pool'] = 'RequestPoolNonThread' if @config["max_request"] <= 1
63
+ @scenarios = []
64
+ 1.upto(@concurrency) do
65
+ @scenarios << @scenario_handler.create_scenario(@suffix)
66
+ end
67
+ @scenario_handler.on_scenarios_created
68
+ $REQUEST_COUNT = 0
69
+ begin
70
+ if @rev_mode
71
+ rev_start
72
+ elsif @config["max_request"] <= 1
73
+ single_start
74
+ else
75
+ thread_start
76
+ end
77
+ rescue SystemExit
78
+ return
79
+ rescue Exception => e
80
+ raise e
81
+ puts "error: #{e.message}"
82
+ puts e.backtrace[0]
83
+ if @fork_mode
84
+ #死ぬ前に親に連絡
85
+ Process::kill(:USR1, Process::ppid)
86
+ else
87
+ report_collect
88
+ end
89
+ end
90
+ end
91
+ def single_start
92
+ th_cnt = 1
93
+ @threads[1] = Thread::current
94
+ @scenarios.each do |sc|
95
+ sc.init
96
+ end
97
+ #シングルスレッドのときは先にプールにためておく
98
+ run_executer th_cnt
99
+ if @failure
100
+ page_check
101
+ exit 1
102
+ else
103
+ exit 0
104
+ end
105
+ end
106
+ def page_check
107
+ keys = @result_record.keys
108
+ pages = []
109
+ keys.each do |key|
110
+ err = @result_record[key][:errorcount] || 0
111
+ if(err > 0)
112
+ pages << key
113
+ end
114
+ end
115
+ unless pages.empty?
116
+ @logger.run("ERROR URL: #{pages.join(',')}\n", 2)
117
+ end
118
+ end
119
+ def thread_start
120
+ @mutex = Mutex::new
121
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
122
+ @cv = ConditionVariable::new
123
+
124
+ @scenarios.each do |sc|
125
+ sc.init
126
+ end
127
+ Thread.abort_on_exception
128
+ @scenario_handler.set_on_turn_complete &method(:on_complete)
129
+ 1.upto(@config["max_request"]) do |th_cnt|
130
+ @threads[th_cnt] = Thread::new(th_cnt) do |th_cnt|
131
+ run_executer th_cnt
132
+ end
133
+ end
134
+ @threads[1].join
135
+ @threads[:main] = Thread::current
136
+ @mutex.synchronize{
137
+ @cv.wait(@mutex)
138
+ }
139
+ @threads.each do |k,th|
140
+ th.kill if th!=Thread::current && k!=:main
141
+ end
142
+ if @failure
143
+ exit 1
144
+ else
145
+ exit 0
146
+ end
147
+ end
148
+ def run_executer th_cnt
149
+ th_id = (sprintf(@id_tmpl, th_cnt) + @suffix).to_i
150
+ report = BaseHandler::get_handler(@config["response_handler"])
151
+ Thread.current[:result] = report
152
+ BaseHandler::set_thread_id(th_id)
153
+ cls = BaseHandler::get_class(@config['scenario_executer'])
154
+ @scenario_handler.set_on_turn_complete(&@register.method(:exec))
155
+ exec = cls::new
156
+ exec.set_on_complete &method(:on_complete)
157
+ @starttime = Time::now
158
+ exec.run
159
+ end
160
+ # = 1プロセス終了時に呼ばれる
161
+ def on_complete
162
+ total = Time::now - @starttime
163
+ @request_pool = BaseHandler::get_handler(@config['request_pool'])
164
+ @request_pool.close
165
+ #puts "thread collect start"
166
+ thread_collect
167
+ #puts "thread collect end"
168
+ report_collect
169
+ #puts "report collect end"
170
+ @logger.run("TotalTime: " + total.to_s + "\n", 2) unless @fork_mode
171
+ @failure = true if $error
172
+ @cv.signal if @cv
173
+ end
174
+ def create_id loop_c, th_c
175
+ (loop_c.to_s + sprintf(@id_tmpl, th_c) + @suffix).to_i
176
+ end
177
+ def thread_collect
178
+ count = 0
179
+ status = '.'
180
+ flag = false
181
+ count = 0
182
+ @threads.each do |k, th|
183
+ if(th[:result])
184
+ status = 't'
185
+ ect = 0
186
+ th[:result].get_record.each do |name,res|
187
+ ect += 1 if res[:errorcount] > 0
188
+ end
189
+ if ect > 0
190
+ status = 'e'
191
+ flag = true unless flag
192
+ end
193
+ #th.kill if th!=Thread::current && k!=:main
194
+ end
195
+ #@logger.run status, 2
196
+ end
197
+ @logger.run "\n", 2 unless @fork_mode
198
+ @logger.run "result: ", 2 unless @fork_mode
199
+ if flag
200
+ @logger.run "failure\n", 2 unless @fork_mode
201
+ else
202
+ @logger.run "success\n", 2 unless @fork_mode
203
+ end
204
+ @failure = flag
205
+ end
206
+ def report_collect
207
+ d = (Time::now - @starttime)*1000
208
+ if @fork_mode && @forkdb
209
+ @forkdb.transaction do
210
+ if not @forkdb['time']
211
+ @forkdb['time'] = d
212
+ elsif @forkdb['time'] < d
213
+ @forkdb['time'] = d
214
+ end
215
+ @forkdb['request_count'] = 0 unless @forkdb['request_count']
216
+ @forkdb['request_count'] += $REQUEST_COUNT
217
+ end
218
+ else
219
+ @logger.run "\nTimePerRequest: #{d / ($REQUEST_COUNT / @config['max_request'])}\n", 2
220
+ @logger.run "RequestPerSec: #{$REQUEST_COUNT / (d/1000)}\n", 2
221
+ end
222
+ record = ResponseRecord::new
223
+ @threads.each do |k, th|
224
+ if th[:result]
225
+ record += th[:result].get_record
226
+ end
227
+ end
228
+ @logger.run("\n\n", 2) unless @fork_mode
229
+ reporter = BaseHandler::get_handler(@config["reporter"])
230
+ reporter.run(record)
231
+ @result_record = record
232
+ end
233
+ def run_fork
234
+ raise "invalid config: fork: nil" unless @config['fork']
235
+ org_reporter = @config['reporter']
236
+
237
+ @logger.run("Process: #{@process_max}\n", 2)
238
+ @parent = false
239
+
240
+ flag = false
241
+ Signal::trap(:USR1){
242
+ return if flag
243
+ flag = true
244
+ #parent
245
+ puts "error: SIGUSR1"
246
+ @pids.each do |pid|
247
+ Process::kill(:USR2, pid)
248
+ end
249
+ }
250
+ Signal::trap(:INT){
251
+ flag = true
252
+ puts "error: SIGINT"
253
+ @pids.each do |pid|
254
+ Process::kill(:USR2, pid)
255
+ end
256
+ }
257
+ Signal::trap(:USR2){
258
+ #child
259
+ report_collect
260
+ exit
261
+ }
262
+
263
+ #各プロセスにはpstoreファイルに集計させる
264
+ @config['reporter'] = 'PStoreReporter'
265
+ fname = 'tmp.db'
266
+ fpath = File::join(@basepath, fname)
267
+ if FileTest::exists?(fpath)
268
+ raise 'cannot create db' unless FileTest::writable?(fpath)
269
+ raise 'cannot create db' unless File::delete(fpath)
270
+ end
271
+ db = PStore::new(fpath)
272
+ @forkdb = db
273
+
274
+ reporter = BaseHandler::get_handler(@config["reporter"])
275
+ reporter.set_dir(@basepath)
276
+ reporter.init_filename(fname)
277
+
278
+ begin
279
+ @pids = []
280
+ @process = @process_max
281
+ digit = @process_max.to_s.size
282
+ tmpl = "%0#{digit}d"
283
+ 1.upto(@process_max) do |i|
284
+ @pids << Process::fork {
285
+ @suffix = sprintf(tmpl, i)
286
+ run_process
287
+ }
288
+ #1プロセス目以外は出力を少なく
289
+ @scenario_handler.silence if i==2
290
+ end
291
+ @forkreport = false
292
+ @org_fork_reporter = org_reporter
293
+ results = Process::waitall
294
+ failure = false
295
+ results.each do |r|
296
+ if r[1].to_i != 0
297
+ failure = true
298
+ break
299
+ end
300
+ end
301
+ @logger.run "\n", 2
302
+ @logger.run "result: ", 2
303
+ unless failure
304
+ @logger.run "success\n", 2
305
+ else
306
+ @logger.run "failure\n", 2
307
+ end
308
+ fork_report
309
+ ensure
310
+ fork_report unless @forkreport
311
+ File::delete(fpath) if File::exists?(fpath)
312
+ end
313
+ end
314
+ def fork_report
315
+ @forkreport = true
316
+ require 'pstore'
317
+ result = nil
318
+ time = nil
319
+ count = nil
320
+ @forkdb.transaction do
321
+ result = @forkdb['result']
322
+ time = @forkdb['time']
323
+ count = @forkdb['request_count']
324
+ end
325
+ @logger.run "\nTimePerRequest: #{time / (count / @config['max_request_show'])}\n", 2 if time && count
326
+ @logger.run "RequestPerSec: #{count / (time/1000)}\n", 2 if time && count
327
+
328
+ reporter = BaseHandler::get_handler(@org_fork_reporter)
329
+ reporter.run(result)
330
+ reporter = nil
331
+ end
332
+ end
333
+ end