jkr 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +81 -0
- data/Guardfile +9 -0
- data/LICENSE.txt +674 -0
- data/{README.txt → README.rdoc} +0 -0
- data/Rakefile +4 -5
- data/bin/console +7 -0
- data/etc/example.plan +32 -0
- data/etc/zsh-comp.sh +82 -0
- data/exe/jkr +6 -0
- data/jkr.gemspec +31 -0
- data/lib/jkr.rb +13 -4
- data/lib/jkr/analysis.rb +5 -14
- data/lib/jkr/analytics.rb +75 -0
- data/lib/jkr/array.rb +47 -0
- data/lib/jkr/blktrace.rb +131 -0
- data/lib/jkr/cli.rb +110 -0
- data/lib/jkr/cpu_usage.rb +81 -0
- data/lib/jkr/cpufreq.rb +201 -0
- data/lib/jkr/dirlock.rb +9 -0
- data/lib/jkr/env.rb +17 -17
- data/lib/jkr/error.rb +5 -0
- data/lib/jkr/numeric.rb +28 -0
- data/lib/jkr/plan.rb +317 -26
- data/lib/jkr/planfinder.rb +40 -0
- data/lib/jkr/plot.rb +626 -0
- data/lib/jkr/stat.rb +2 -0
- data/lib/jkr/stat/kmeans-1d.rb +94 -0
- data/lib/jkr/su_cmd +163 -0
- data/lib/jkr/sysinfo.rb +34 -0
- data/lib/jkr/trial.rb +91 -16
- data/lib/jkr/userutils.rb +300 -22
- data/lib/jkr/utils.rb +38 -314
- data/lib/jkr/version.rb +3 -0
- data/sample-jkr.plan +52 -0
- metadata +171 -63
- data/bin/jkr +0 -224
- data/test/test_jkr.rb +0 -8
data/lib/jkr/utils.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
|
2
2
|
require 'fileutils'
|
3
|
-
require 'popen4'
|
4
3
|
require 'thread'
|
5
4
|
|
6
5
|
require 'jkr/userutils'
|
@@ -27,7 +26,7 @@ class Barrier
|
|
27
26
|
end
|
28
27
|
end
|
29
28
|
|
30
|
-
|
29
|
+
module Jkr
|
31
30
|
class Utils
|
32
31
|
def self.reserve_next_dir(dir, suffix = "")
|
33
32
|
dirs = Dir.glob("#{dir}#{File::SEPARATOR}???*")
|
@@ -37,341 +36,55 @@ class Jkr
|
|
37
36
|
max_num = [$~[0].to_i, max_num].max
|
38
37
|
end
|
39
38
|
end
|
40
|
-
|
39
|
+
|
41
40
|
num = max_num + 1
|
42
|
-
dir = "#{dir}#{File::SEPARATOR}" + sprintf("%
|
41
|
+
dir = "#{dir}#{File::SEPARATOR}" + sprintf("%05d%s", num, suffix)
|
43
42
|
FileUtils.mkdir(dir)
|
44
43
|
dir
|
45
44
|
end
|
46
45
|
end
|
47
46
|
|
48
|
-
module PlanUtils
|
49
|
-
# info about processes spawned by me
|
50
|
-
def procdb
|
51
|
-
@procdb ||= Hash.new
|
52
|
-
end
|
53
|
-
def procdb_spawn(pid, command, owner_thread)
|
54
|
-
@procdb_mutex.synchronize do
|
55
|
-
self.procdb[pid] = {
|
56
|
-
:pid => pid,
|
57
|
-
:command => command,
|
58
|
-
:thread => owner_thread,
|
59
|
-
:status => nil
|
60
|
-
}
|
61
|
-
end
|
62
|
-
end
|
63
|
-
def procdb_waitpid(pid)
|
64
|
-
t = nil
|
65
|
-
@procdb_mutex.synchronize do
|
66
|
-
if self.procdb[pid]
|
67
|
-
t = self.procdb[pid][:thread]
|
68
|
-
end
|
69
|
-
end
|
70
|
-
t.join if t
|
71
|
-
end
|
72
|
-
def procdb_resetpid(pid)
|
73
|
-
@procdb_mutex.synchronize do
|
74
|
-
if self.procdb[pid]
|
75
|
-
self.procdb.delete(pid)
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
def procdb_update_status(pid, status)
|
80
|
-
@procdb_mutex.synchronize do
|
81
|
-
if self.procdb[pid]
|
82
|
-
self.procdb[pid][:status] = status
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
86
|
-
def procdb_get(pid)
|
87
|
-
@procdb_mutex.synchronize do
|
88
|
-
self.procdb[pid]
|
89
|
-
end
|
90
|
-
end
|
91
|
-
def procdb_get_status(pid)
|
92
|
-
proc = self.procdb_get(pid)
|
93
|
-
proc && proc[:status]
|
94
|
-
end
|
95
|
-
def procdb_get_command(pid)
|
96
|
-
proc = self.procdb_get(pid)
|
97
|
-
proc && proc[:command]
|
98
|
-
end
|
99
|
-
|
100
|
-
def cmd(*args)
|
101
|
-
@procdb_mutex ||= Mutex.new
|
102
|
-
options = (if args.last.is_a? Hash
|
103
|
-
args.pop
|
104
|
-
else
|
105
|
-
{}
|
106
|
-
end)
|
107
|
-
options = {
|
108
|
-
:wait => true,
|
109
|
-
:timeout => 0,
|
110
|
-
:raise_failure => true,
|
111
|
-
:stdin => nil,
|
112
|
-
:stdout => [$stdout],
|
113
|
-
:stderr => [$stderr]
|
114
|
-
}.merge(options)
|
115
|
-
|
116
|
-
if options[:timeout] > 0 && ! options[:wait]
|
117
|
-
raise ArgumentError.new("cmd: 'wait' must be true if 'timeout' specified.")
|
118
|
-
end
|
119
|
-
|
120
|
-
start_time = Time.now
|
121
|
-
pid = nil
|
122
|
-
status = nil
|
123
|
-
args.flatten!
|
124
|
-
args.map!(&:to_s)
|
125
|
-
command = args.join(" ")
|
126
|
-
barrier = Barrier.new(2)
|
127
|
-
process_exited = false
|
128
|
-
|
129
|
-
t = Thread.new {
|
130
|
-
pipers = []
|
131
|
-
status = POpen4::popen4(command){|p_stdout, p_stderr, p_stdin, p_id|
|
132
|
-
pid = p_id
|
133
|
-
barrier.wait
|
134
|
-
stdouts = if options[:stdout].is_a? Array
|
135
|
-
options[:stdout]
|
136
|
-
else
|
137
|
-
[options[:stdout]]
|
138
|
-
end
|
139
|
-
stderrs = if options[:stderr].is_a? Array
|
140
|
-
options[:stderr]
|
141
|
-
else
|
142
|
-
[options[:stderr]]
|
143
|
-
end
|
144
|
-
pipers << Thread.new{
|
145
|
-
target = p_stdout
|
146
|
-
timeout_count = 0
|
147
|
-
while true
|
148
|
-
begin
|
149
|
-
if (ready = IO.select([target], [], [], 1))
|
150
|
-
ready.first.each do |fd|
|
151
|
-
buf = fd.read_nonblock(4096)
|
152
|
-
stdouts.each{|out| out.print buf}
|
153
|
-
end
|
154
|
-
Thread.exit if target.eof?
|
155
|
-
else
|
156
|
-
if process_exited
|
157
|
-
timeout_count += 1
|
158
|
-
if timeout_count > 5
|
159
|
-
target.close_read
|
160
|
-
Thread.exit
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
rescue IOError => err
|
165
|
-
if target.closed?
|
166
|
-
Thread.exit
|
167
|
-
end
|
168
|
-
end
|
169
|
-
end
|
170
|
-
}
|
171
|
-
pipers << Thread.new{
|
172
|
-
target = p_stderr
|
173
|
-
timeout_count = 0
|
174
|
-
while true
|
175
|
-
begin
|
176
|
-
if (ready = IO.select([target], [], [], 1))
|
177
|
-
ready.first.each do |fd|
|
178
|
-
buf = fd.read_nonblock(4096)
|
179
|
-
stderrs.each{|out| out.print buf}
|
180
|
-
end
|
181
|
-
Thread.exit if target.eof?
|
182
|
-
else
|
183
|
-
if process_exited
|
184
|
-
timeout_count += 1
|
185
|
-
if timeout_count > 5
|
186
|
-
target.close_read
|
187
|
-
Thread.exit
|
188
|
-
end
|
189
|
-
end
|
190
|
-
end
|
191
|
-
rescue IOError => err
|
192
|
-
if target.closed?
|
193
|
-
Thread.exit
|
194
|
-
end
|
195
|
-
end
|
196
|
-
end
|
197
|
-
}
|
198
|
-
if options[:stdin]
|
199
|
-
pipers << Thread.new{
|
200
|
-
target = options[:stdin]
|
201
|
-
timeout_count = 0
|
202
|
-
while true
|
203
|
-
begin
|
204
|
-
if (ready = IO.select([target], [], [], 1))
|
205
|
-
ready.first.each do |fd|
|
206
|
-
buf = fd.read_nonblock(4096)
|
207
|
-
p_stdin.print buf
|
208
|
-
end
|
209
|
-
else
|
210
|
-
if process_exited
|
211
|
-
timeout_count += 1
|
212
|
-
if timeout_count > 5
|
213
|
-
p_stdin.close_write
|
214
|
-
Thread.exit
|
215
|
-
end
|
216
|
-
end
|
217
|
-
end
|
218
|
-
rescue IOError => err
|
219
|
-
if target.closed?
|
220
|
-
Thread.exit
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|
224
|
-
}
|
225
|
-
end
|
226
|
-
}
|
227
|
-
pipers.each{|t| t.join}
|
228
|
-
raise ArgumentError.new("Invalid command: #{command}") unless status
|
229
|
-
procdb_update_status(pid, status)
|
230
|
-
}
|
231
|
-
barrier.wait
|
232
|
-
procdb_spawn(pid, command, t)
|
233
|
-
timekeeper = nil
|
234
|
-
|
235
|
-
killed = false
|
236
|
-
timekeeper = nil
|
237
|
-
if options[:timeout] > 0
|
238
|
-
timekeeper = Thread.new do
|
239
|
-
sleep(options[:timeout])
|
240
|
-
begin
|
241
|
-
Process.kill(:INT, pid)
|
242
|
-
killed = true
|
243
|
-
rescue Errno::ESRCH # No such process
|
244
|
-
end
|
245
|
-
end
|
246
|
-
end
|
247
|
-
if options[:wait]
|
248
|
-
timekeeper.join if timekeeper
|
249
|
-
t.join
|
250
|
-
if (! killed) && options[:raise_failure] && status.exitstatus != 0
|
251
|
-
raise RuntimeError.new("'#{command}' failed.")
|
252
|
-
end
|
253
|
-
end
|
254
|
-
while ! pid
|
255
|
-
sleep 0.001 # do nothing
|
256
|
-
end
|
257
|
-
|
258
|
-
pid
|
259
|
-
end
|
260
|
-
|
261
|
-
def with_process2(*args)
|
262
|
-
options = (if args.last.is_a? Hash
|
263
|
-
args.pop
|
264
|
-
else
|
265
|
-
{}
|
266
|
-
end )
|
267
|
-
options = {
|
268
|
-
:kill_on_exit => false
|
269
|
-
}.merge(options)
|
270
|
-
|
271
|
-
command = args.flatten.map(&:to_s).join(" ")
|
272
|
-
pid = Process.spawn(command)
|
273
|
-
|
274
|
-
err = nil
|
275
|
-
begin
|
276
|
-
yield
|
277
|
-
rescue Exception => e
|
278
|
-
err = e
|
279
|
-
end
|
280
|
-
|
281
|
-
if options[:kill_on_exit]
|
282
|
-
Process.kill(:INT, pid)
|
283
|
-
else
|
284
|
-
if err
|
285
|
-
begin
|
286
|
-
Process.kill(:TERM, pid)
|
287
|
-
rescue Exception
|
288
|
-
end
|
289
|
-
else
|
290
|
-
begin
|
291
|
-
status = Process.waitpid(pid)
|
292
|
-
p status
|
293
|
-
rescue Errno::ESRCH
|
294
|
-
end
|
295
|
-
end
|
296
|
-
end
|
297
|
-
raise err if err
|
298
|
-
end
|
299
|
-
|
300
|
-
def with_process(*args)
|
301
|
-
options = (if args.last.is_a? Hash
|
302
|
-
args.pop
|
303
|
-
else
|
304
|
-
{}
|
305
|
-
end )
|
306
|
-
options = {
|
307
|
-
:kill_on_exit => false
|
308
|
-
}.merge(options)
|
309
|
-
options[:wait] = false
|
310
|
-
|
311
|
-
args.push(options)
|
312
|
-
pid = cmd(*args)
|
313
|
-
|
314
|
-
err = nil
|
315
|
-
begin
|
316
|
-
yield
|
317
|
-
rescue Exception => e
|
318
|
-
err = e
|
319
|
-
end
|
320
|
-
|
321
|
-
if options[:kill_on_exit]
|
322
|
-
Process.kill(:INT, pid)
|
323
|
-
else
|
324
|
-
if err
|
325
|
-
begin
|
326
|
-
Process.kill(:TERM, pid)
|
327
|
-
rescue Exception
|
328
|
-
end
|
329
|
-
else
|
330
|
-
procdb_waitpid(pid)
|
331
|
-
status = procdb_get_status(pid)
|
332
|
-
unless status && status.exitstatus == 0
|
333
|
-
command = procdb_get_command(pid) || "Unknown command"
|
334
|
-
raise RuntimeError.new("'#{command}' failed.")
|
335
|
-
end
|
336
|
-
procdb_resetpid(pid)
|
337
|
-
end
|
338
|
-
end
|
339
|
-
|
340
|
-
raise err if err
|
341
|
-
end
|
342
|
-
|
343
|
-
def use_script(name)
|
344
|
-
name = name.to_s
|
345
|
-
name = name + ".rb" unless name =~ /\.rb$/
|
346
|
-
dir = @plan.jkr_env.jkr_script_dir
|
347
|
-
path = File.join(dir, name)
|
348
|
-
script = File.open(path, "r").read
|
349
|
-
self.instance_eval(script, path, 1)
|
350
|
-
true
|
351
|
-
end
|
352
|
-
end
|
353
|
-
|
354
47
|
class TrialUtils
|
355
48
|
def self.undef_routine_utils(plan)
|
356
|
-
plan
|
49
|
+
_plan = plan
|
50
|
+
begin
|
51
|
+
if _plan.routine
|
52
|
+
_plan.routine.binding.eval <<EOS
|
357
53
|
undef result_file
|
358
54
|
undef result_file_name
|
55
|
+
undef rname
|
56
|
+
undef common_file_name
|
57
|
+
undef cname
|
359
58
|
undef touch_result_file
|
360
59
|
undef with_result_file
|
361
60
|
EOS
|
61
|
+
end
|
62
|
+
end while _plan = _plan.base_plan
|
362
63
|
end
|
363
64
|
|
364
65
|
def self.define_routine_utils(result_dir, plan, params)
|
66
|
+
_plan = plan
|
67
|
+
begin
|
365
68
|
line = __LINE__; src = <<EOS
|
366
69
|
def result_file_name(basename)
|
367
70
|
File.join(#{result_dir.inspect}, basename)
|
368
71
|
end
|
72
|
+
def rname(basename)
|
73
|
+
result_file_name(basename)
|
74
|
+
end
|
369
75
|
|
370
76
|
def result_file(basename, mode = "a+")
|
371
77
|
path = result_file_name(basename)
|
372
78
|
File.open(path, mode)
|
373
79
|
end
|
374
80
|
|
81
|
+
def common_file_name(basename)
|
82
|
+
File.join(File.dirname(#{result_dir.inspect}), basename)
|
83
|
+
end
|
84
|
+
def cname(basename)
|
85
|
+
result_file_name(basename)
|
86
|
+
end
|
87
|
+
|
375
88
|
def touch_result_file(basename, options = {})
|
376
89
|
path = result_file_name(basename)
|
377
90
|
FileUtils.touch(path, options)
|
@@ -391,7 +104,10 @@ def with_result_file(basename, mode = "a+")
|
|
391
104
|
file.path
|
392
105
|
end
|
393
106
|
EOS
|
394
|
-
|
107
|
+
if _plan.routine
|
108
|
+
_plan.routine.binding.eval(src, __FILE__, line)
|
109
|
+
end
|
110
|
+
end while _plan = _plan.base_plan
|
395
111
|
end
|
396
112
|
end
|
397
113
|
|
@@ -414,7 +130,7 @@ def resultset()
|
|
414
130
|
dirs = Dir.glob(File.join(#{resultset_dir.inspect}, "*"))
|
415
131
|
dirs.map{|dir| File.basename dir}.select{|basename|
|
416
132
|
basename =~ /\\A\\d{3,}\\Z/
|
417
|
-
}
|
133
|
+
}.sort
|
418
134
|
end
|
419
135
|
|
420
136
|
def result_file_name(num, basename)
|
@@ -423,6 +139,9 @@ def result_file_name(num, basename)
|
|
423
139
|
end
|
424
140
|
File.join(#{resultset_dir.inspect}, num, basename)
|
425
141
|
end
|
142
|
+
def rname(num, basename)
|
143
|
+
result_file_name(num, basename)
|
144
|
+
end
|
426
145
|
|
427
146
|
def result_file(num, basename, mode = "r")
|
428
147
|
path = result_file_name(num, basename)
|
@@ -432,6 +151,9 @@ end
|
|
432
151
|
def common_file_name(basename)
|
433
152
|
File.join(#{resultset_dir.inspect}, basename)
|
434
153
|
end
|
154
|
+
def cname(basename)
|
155
|
+
common_file_name(basename)
|
156
|
+
end
|
435
157
|
|
436
158
|
def common_file(basename, mode = "r")
|
437
159
|
path = common_file_name(basename)
|
@@ -464,7 +186,9 @@ def with_result_file(basename, mode = "r")
|
|
464
186
|
file.path
|
465
187
|
end
|
466
188
|
EOS
|
467
|
-
plan.
|
189
|
+
if plan.analysis
|
190
|
+
plan.analysis.binding.eval(src, __FILE__, line)
|
191
|
+
end
|
468
192
|
end
|
469
193
|
end
|
470
194
|
end
|
data/lib/jkr/version.rb
ADDED
data/sample-jkr.plan
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# -*- mode: ruby -*-
|
2
|
+
|
3
|
+
TERMWIDTH = 80
|
4
|
+
|
5
|
+
title "Default experiment plan"
|
6
|
+
description "This is a default experiment plan specification file."
|
7
|
+
|
8
|
+
def_parameters do
|
9
|
+
# use 'variable' for defining variables
|
10
|
+
variable :foo => ["fooA", "fooB"]
|
11
|
+
variable :bar => 1..5
|
12
|
+
|
13
|
+
# use 'parameter' for defining static parameters
|
14
|
+
parameter :baz => "baz"
|
15
|
+
end
|
16
|
+
|
17
|
+
# preparation
|
18
|
+
def_prep do |plan|
|
19
|
+
puts '*' * TERMWIDTH
|
20
|
+
puts "Start execution: #{plan.title}"
|
21
|
+
end
|
22
|
+
|
23
|
+
# routine executed for each combination of values of variables
|
24
|
+
def_routine do |plan, params|
|
25
|
+
puts '-' * TERMWIDTH
|
26
|
+
|
27
|
+
# the value of "parameter/variable :foo" can be obtained by
|
28
|
+
# "param[:foo]"
|
29
|
+
puts "hello world, #{params[:foo]}"
|
30
|
+
puts "hello world, #{params[:bar]}"
|
31
|
+
|
32
|
+
puts(params[:foo] + params[:baz])
|
33
|
+
|
34
|
+
touch = touch_result_file("touched-file.log")
|
35
|
+
|
36
|
+
with_result_file("test.log") do |test_log|
|
37
|
+
test_log.puts "hello world, #{params[:foo]}"
|
38
|
+
test_log.puts "hello world, #{params[:bar]}"
|
39
|
+
test_log.puts(params[:foo] + params[:baz])
|
40
|
+
[$stdout, test_log].map{|io| io.puts "touched #{touch}"}
|
41
|
+
end
|
42
|
+
|
43
|
+
puts '-' * TERMWIDTH
|
44
|
+
end
|
45
|
+
|
46
|
+
# cleanup
|
47
|
+
def_cleanup do |plan|
|
48
|
+
puts '*' * TERMWIDTH
|
49
|
+
puts "Finished execution: #{plan.title}"
|
50
|
+
puts "FileUtils.remove(#{plan.file_path.inspect})"
|
51
|
+
# FileUtils.remove(plan.file_path)
|
52
|
+
end
|