nera 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -0
- data/Manifest.txt +37 -0
- data/README.rdoc +47 -0
- data/Rakefile +27 -0
- data/bin/nera +29 -0
- data/bin/nera_addsim +30 -0
- data/lib/nera/nera_cui.rb +417 -0
- data/lib/nera/nera_database.rb +281 -0
- data/lib/nera/nera_db_folders.rb +226 -0
- data/lib/nera/nera_dialog.rb +205 -0
- data/lib/nera/nera_job_layer_controller.rb +237 -0
- data/lib/nera/nera_job_records.rb +111 -0
- data/lib/nera/nera_job_script.rb +202 -0
- data/lib/nera/nera_parameter_layer_controller.rb +157 -0
- data/lib/nera/nera_parameter_records.rb +186 -0
- data/lib/nera/nera_run_layer_controller.rb +192 -0
- data/lib/nera/nera_run_records.rb +184 -0
- data/lib/nera/nera_simulator.rb +26 -0
- data/lib/nera/nera_simulator_layer_controller.rb +66 -0
- data/lib/nera/nera_simulator_records.rb +84 -0
- data/lib/nera.rb +25 -0
- data/lib/nera_addsim/make_simulator.rb +307 -0
- data/scripts/make_manifest.rb +21 -0
- data/test/runner.rb +3 -0
- data/test/test_helper.rb +52 -0
- data/test/test_nera_database.rb +221 -0
- data/test/test_nera_db_folders.rb +209 -0
- data/test/test_nera_dialog.rb +134 -0
- data/test/test_nera_job_layer_controller.rb +132 -0
- data/test/test_nera_job_records.rb +260 -0
- data/test/test_nera_parameter_layer_controller.rb +188 -0
- data/test/test_nera_parameter_records.rb +285 -0
- data/test/test_nera_run_layer_controller.rb +171 -0
- data/test/test_nera_run_records.rb +290 -0
- data/test/test_nera_simulator.rb +26 -0
- data/test/test_nera_simulator_layer_controller.rb +54 -0
- data/test/test_nera_simulator_records.rb +140 -0
- metadata +125 -0
@@ -0,0 +1,237 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'yaml'
|
4
|
+
require 'nera_db_folders'
|
5
|
+
require 'nera_simulator_records'
|
6
|
+
require 'nera_parameter_records'
|
7
|
+
require 'nera_run_records'
|
8
|
+
require 'nera_job_records'
|
9
|
+
require 'nera_job_script'
|
10
|
+
|
11
|
+
module NERA
|
12
|
+
|
13
|
+
class JobLayerController
|
14
|
+
|
15
|
+
# instance of NERA::DbFolders
|
16
|
+
@db_folders
|
17
|
+
# instance of NERA::JobRecords
|
18
|
+
@job_records
|
19
|
+
|
20
|
+
def initialize( path_db_folder)
|
21
|
+
raise ArgumentError unless path_db_folder.is_a?(String)
|
22
|
+
|
23
|
+
@db_folders = NERA::DbFolders.new( path_db_folder)
|
24
|
+
@job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
25
|
+
end
|
26
|
+
|
27
|
+
public
|
28
|
+
def path_to_job_layer
|
29
|
+
return @db_folders.path_to_job_layer
|
30
|
+
end
|
31
|
+
|
32
|
+
def not_finished_list_in_csv
|
33
|
+
list = @job_records.list_all
|
34
|
+
header = @job_records.keys.dup
|
35
|
+
header_ab = "id, state, simulator, param_id, run_id, num_runs, created_at, updated_at, host_name"
|
36
|
+
|
37
|
+
csv_list = []
|
38
|
+
list.each do |r_hash|
|
39
|
+
strings = []
|
40
|
+
header.each do |key|
|
41
|
+
if r_hash[key].is_a?(DateTime)
|
42
|
+
strings << r_hash[key].to_s.split('T')[0]
|
43
|
+
else
|
44
|
+
strings << r_hash[key].to_s
|
45
|
+
end
|
46
|
+
end
|
47
|
+
csv_list << strings.join(", ")
|
48
|
+
end
|
49
|
+
return header_ab, csv_list
|
50
|
+
end
|
51
|
+
|
52
|
+
def cancel_jobs( job_ids)
|
53
|
+
unless job_ids.is_a?(Array)
|
54
|
+
raise ArgumentError, "job_ids must be an Array."
|
55
|
+
end
|
56
|
+
job_ids.each do |x|
|
57
|
+
raise ArgumentError, "each element of job_ids must be an Integer" unless x.is_a?(Integer)
|
58
|
+
end
|
59
|
+
|
60
|
+
flag = nil
|
61
|
+
@job_records.transaction {
|
62
|
+
job_ids.each do |jid|
|
63
|
+
d = @job_records.destroy( jid)
|
64
|
+
next unless d
|
65
|
+
sim_recs = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
66
|
+
klass = eval( d[:simulator])
|
67
|
+
raise RuntimeError, "must not happen" unless klass
|
68
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( klass, d[:parameter_id]) )
|
69
|
+
run_records.transaction {
|
70
|
+
a = run_records.destroy_job_id( jid)
|
71
|
+
raise RuntimeError, "must not happen" unless a
|
72
|
+
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
73
|
+
}
|
74
|
+
File.open( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]), 'w') do |io|
|
75
|
+
YAML.dump( run_records.list_all, io)
|
76
|
+
io.flush
|
77
|
+
end
|
78
|
+
flag = true
|
79
|
+
end
|
80
|
+
}
|
81
|
+
dump_in_yaml
|
82
|
+
return flag
|
83
|
+
end
|
84
|
+
|
85
|
+
def include_list
|
86
|
+
list = @db_folders.search_include_files
|
87
|
+
list.map! do |path|
|
88
|
+
path.sub( @db_folders.path_to_include_layer, '')
|
89
|
+
end
|
90
|
+
return list
|
91
|
+
end
|
92
|
+
|
93
|
+
def include( filename)
|
94
|
+
@job_records.transaction {
|
95
|
+
fpath = @db_folders.path_to_include_layer + filename
|
96
|
+
return nil unless File.exist?(fpath)
|
97
|
+
jinfo, rinfo = expand_and_parse( fpath)
|
98
|
+
run_layer_path, run_recs, run_yaml_path = check_consistency( jinfo, rinfo)
|
99
|
+
run_recs.transaction {
|
100
|
+
folder_path = fpath.sub(/.tar.bz2$/,'/')
|
101
|
+
file_move( folder_path, run_layer_path, jinfo[:run_ids])
|
102
|
+
update_tables( run_recs, jinfo, rinfo)
|
103
|
+
}
|
104
|
+
remove_files( fpath)
|
105
|
+
File.open( run_yaml_path, 'w') do |io|
|
106
|
+
YAML.dump( run_recs.list_all, io)
|
107
|
+
io.flush
|
108
|
+
end
|
109
|
+
}
|
110
|
+
dump_in_yaml
|
111
|
+
return true
|
112
|
+
end
|
113
|
+
|
114
|
+
private
|
115
|
+
def expand_and_parse( fpath)
|
116
|
+
bz2_path = fpath.sub( /^#{@db_folders.path_to_include_layer}/, '')
|
117
|
+
Dir.chdir( @db_folders.path_to_include_layer) {
|
118
|
+
cmd = "bunzip2 #{bz2_path}"
|
119
|
+
system(cmd)
|
120
|
+
raise "command : #{cmd} failed" unless $? == 0
|
121
|
+
tarpath = bz2_path.sub(/.bz2$/,'')
|
122
|
+
cmd = "tar xvf #{tarpath}"
|
123
|
+
system(cmd)
|
124
|
+
raise "command : #{cmd} failed" unless $? == 0
|
125
|
+
stat_path = tarpath.sub(/.tar$/,'/nera_status.rb')
|
126
|
+
js = NERA::JobScript.new(@db_folders)
|
127
|
+
jinfo, rinfo = js.parse_status( stat_path)
|
128
|
+
return jinfo, rinfo
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
private
|
133
|
+
def check_consistency( jinfo, rinfo)
|
134
|
+
# record exists?
|
135
|
+
found = @job_records.find_by_id( jinfo[:job_id] )
|
136
|
+
return nil unless found
|
137
|
+
|
138
|
+
# check consistency of the simulator
|
139
|
+
conflicting unless jinfo[:simulator] == found[:simulator]
|
140
|
+
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
141
|
+
sim_class = eval( jinfo[:simulator])
|
142
|
+
conflicting unless jinfo[:simulator] == sim_class.to_s
|
143
|
+
|
144
|
+
# check consistency of the parameter
|
145
|
+
conflicting unless jinfo[:param_id] == found[:parameter_id]
|
146
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class) , sim_class)
|
147
|
+
param = param_records.find_by_id( jinfo[:param_id])
|
148
|
+
jinfo[:parameters].each_pair do |key, value|
|
149
|
+
conflicting unless param[key] == value
|
150
|
+
end
|
151
|
+
|
152
|
+
# check consistency of the runs
|
153
|
+
conflicting unless jinfo[:run_ids][0] == found[:run_id]
|
154
|
+
conflicting unless jinfo[:run_ids].size == found[:number_of_runs]
|
155
|
+
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]) )
|
156
|
+
run_yaml_path = @db_folders.path_to_runs_yaml( sim_class, jinfo[:param_id])
|
157
|
+
found_runs = run_recs.list_all_not_finished.find_all do |rec|
|
158
|
+
rec[:job_id] == jinfo[:job_id] and jinfo[:run_ids].include?(rec[:id])
|
159
|
+
end
|
160
|
+
conflicting unless jinfo[:run_ids].size == found_runs.size
|
161
|
+
|
162
|
+
# check consistency of run_info
|
163
|
+
raise "Run info is not consistent with job info!" unless jinfo[:run_ids] == rinfo.keys.sort
|
164
|
+
|
165
|
+
run_layer_path = @db_folders.path_to_run_layer( sim_class, jinfo[:param_id])
|
166
|
+
return run_layer_path, run_recs, run_yaml_path
|
167
|
+
end
|
168
|
+
|
169
|
+
def conflicting
|
170
|
+
raise RuntimeError, "This information is conflicting with the table."
|
171
|
+
end
|
172
|
+
|
173
|
+
def file_move( folder_path, run_layer_path, run_ids)
|
174
|
+
run_ids.each do |rid|
|
175
|
+
data_folder = folder_path + rid.to_s
|
176
|
+
raise "must not happen" unless FileTest.directory?(data_folder)
|
177
|
+
data_files = Dir.glob(data_folder + '/*')
|
178
|
+
data_files.each do |data_file|
|
179
|
+
dest = run_layer_path + sprintf("%06d_",rid) + File.basename( data_file)
|
180
|
+
raise "file already exists." if File.exists?(dest)
|
181
|
+
FileUtils.mv( data_file, dest, {:verbose => true} )
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
def update_tables( run_recs, jinfo, rinfos)
|
187
|
+
d = DateTime.now
|
188
|
+
rinfos.each_pair do |rid, info|
|
189
|
+
f = run_recs.update_state_to_finished( rid, info[:start_at], info[:finish_at], d,
|
190
|
+
info[:real_time], info[:user_time], jinfo[:host_name] )
|
191
|
+
raise "must not happen" unless f
|
192
|
+
end
|
193
|
+
destroyed = @job_records.destroy( jinfo[:job_id])
|
194
|
+
destroyed[:simulator] = jinfo[:simulator]
|
195
|
+
destroyed[:parameters] = jinfo[:parameters]
|
196
|
+
destroyed[:start_at] = jinfo[:start_at]
|
197
|
+
destroyed[:host_name] = jinfo[:host_name]
|
198
|
+
destroyed[:finish_at] = jinfo[:finish_at]
|
199
|
+
File.open( @db_folders.path_to_finished_jobs_file, "a") do |io|
|
200
|
+
YAML.dump( destroyed, io)
|
201
|
+
io.flush
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def remove_files( fpath)
|
206
|
+
tar_path = fpath.sub(/.bz2$/,'')
|
207
|
+
FileUtils.rm( tar_path)
|
208
|
+
fold_path = tar_path.sub(/.tar$/,'/')
|
209
|
+
FileUtils.rm_r( fold_path)
|
210
|
+
sh_path = @db_folders.path_to_job_layer + File.basename( tar_path.sub(/.tar$/,'.sh'))
|
211
|
+
if FileTest.exist?(sh_path)
|
212
|
+
ff = @db_folders.path_to_job_layer + 'finished_jobs/'
|
213
|
+
FileUtils.mkdir_p( ff)
|
214
|
+
FileUtils.mv(sh_path, ff )
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
public
|
219
|
+
def include_all
|
220
|
+
flag = true
|
221
|
+
include_list.each do |fname|
|
222
|
+
f = include(fname)
|
223
|
+
flag = nil unless f
|
224
|
+
end
|
225
|
+
dump_in_yaml
|
226
|
+
return flag
|
227
|
+
end
|
228
|
+
|
229
|
+
def dump_in_yaml
|
230
|
+
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
231
|
+
YAML.dump( @job_records.list_all, io)
|
232
|
+
io.flush
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
end
|
237
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'nera_database'
|
2
|
+
|
3
|
+
module NERA
|
4
|
+
|
5
|
+
# This class access the simulator table.
|
6
|
+
class JobRecords
|
7
|
+
|
8
|
+
# keys of the simulator table
|
9
|
+
ATTRIBUTES = [ [:id, Integer],
|
10
|
+
[:state, Symbol],
|
11
|
+
[:simulator, String],
|
12
|
+
[:parameter_id, Integer],
|
13
|
+
[:run_id, Integer],
|
14
|
+
[:number_of_runs, Integer],
|
15
|
+
[:created_at, DateTime],
|
16
|
+
[:updated_at, DateTime],
|
17
|
+
[:host_name, String]
|
18
|
+
]
|
19
|
+
|
20
|
+
# NERA::Database object
|
21
|
+
@db
|
22
|
+
|
23
|
+
# db_file is the path to database file
|
24
|
+
def initialize( db_file)
|
25
|
+
@db = NERA::Database.new( db_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
# if file already exists, returns false
|
29
|
+
def self.create_table( db_file)
|
30
|
+
NERA::Database.create_table( db_file)
|
31
|
+
end
|
32
|
+
|
33
|
+
# return keys of the simulator table
|
34
|
+
def keys
|
35
|
+
keys = ATTRIBUTES.map do |x|
|
36
|
+
x[0]
|
37
|
+
end
|
38
|
+
return keys
|
39
|
+
end
|
40
|
+
|
41
|
+
# sim_class must be a subclass of NERA::Simulator
|
42
|
+
def add( sim_class, param_id, r_id, num_runs)
|
43
|
+
unless sim_class.is_a?(Class) and sim_class.superclass == NERA::Simulator and param_id.is_a?(Integer) and r_id.is_a?(Integer) and num_runs.is_a?(Integer)
|
44
|
+
raise ArgumentError, "Each argument must be an Integer"
|
45
|
+
end
|
46
|
+
if param_id <= 0 or r_id <= 0 or num_runs <= 0
|
47
|
+
raise ArgumentError, "Each argument must be a positive integer."
|
48
|
+
end
|
49
|
+
d = DateTime.now
|
50
|
+
h = { :state => :created, :simulator => sim_class.to_s, :parameter_id => param_id, :run_id => r_id,
|
51
|
+
:number_of_runs => num_runs, :created_at => d, :updated_at => d, :host_name => nil}
|
52
|
+
id = @db.add( h)
|
53
|
+
return id
|
54
|
+
end
|
55
|
+
|
56
|
+
def list_all
|
57
|
+
l = @db.find_all do |r| true end
|
58
|
+
return l.to_a
|
59
|
+
end
|
60
|
+
|
61
|
+
def list_created
|
62
|
+
l = @db.find_all do |r|
|
63
|
+
r[:state] == :created
|
64
|
+
end
|
65
|
+
return l.to_a
|
66
|
+
end
|
67
|
+
|
68
|
+
def find_by_id( i)
|
69
|
+
@db.find_by_id( i)
|
70
|
+
end
|
71
|
+
|
72
|
+
def update_to_state_copied( id, hostname)
|
73
|
+
unless id.is_a?(Integer) and hostname.is_a?(String)
|
74
|
+
raise ArgumentError, "Type of the argument is invalid."
|
75
|
+
end
|
76
|
+
|
77
|
+
found = @db.find_by_id( id)
|
78
|
+
return nil if found == nil
|
79
|
+
|
80
|
+
found[:state] = :copied
|
81
|
+
found[:updated_at] = DateTime.now
|
82
|
+
found[:host_name] = hostname
|
83
|
+
@db.update(found)
|
84
|
+
end
|
85
|
+
|
86
|
+
def update_to_state_submitted( id, hostname)
|
87
|
+
unless id.is_a?(Integer) and hostname.is_a?(String)
|
88
|
+
raise ArgumentError, "Type of the argument is invalid."
|
89
|
+
end
|
90
|
+
|
91
|
+
found = @db.find_by_id( id)
|
92
|
+
return nil if found == nil
|
93
|
+
|
94
|
+
found[:state] = :submitted
|
95
|
+
found[:updated_at] = DateTime.now
|
96
|
+
found[:host_name] = hostname
|
97
|
+
@db.update(found)
|
98
|
+
end
|
99
|
+
|
100
|
+
def destroy( id)
|
101
|
+
raise ArgumentError unless id.is_a?(Integer)
|
102
|
+
@db.destroy(id)
|
103
|
+
end
|
104
|
+
|
105
|
+
def transaction
|
106
|
+
@db.transaction{
|
107
|
+
yield
|
108
|
+
}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,202 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'date'
|
3
|
+
require 'nera_db_folders'
|
4
|
+
require 'nera_simulator'
|
5
|
+
|
6
|
+
|
7
|
+
module NERA
|
8
|
+
|
9
|
+
# This class create & parse job scripts
|
10
|
+
class JobScript
|
11
|
+
|
12
|
+
# NERA::DbFolders object
|
13
|
+
@db_folder
|
14
|
+
|
15
|
+
#--------------------------------------
|
16
|
+
#-class methods -----------------------
|
17
|
+
#--------------------------------------
|
18
|
+
|
19
|
+
# db_folder is the instance of DbFolders
|
20
|
+
def initialize( db_folder)
|
21
|
+
unless db_folder.instance_of?(NERA::DbFolders) then
|
22
|
+
raise ArgumentError, "Argument must be an instance of NERA::DbFolders."
|
23
|
+
end
|
24
|
+
@db_folder = db_folder
|
25
|
+
end
|
26
|
+
|
27
|
+
# job script
|
28
|
+
def job_script_body(job_info)
|
29
|
+
############## job_script ###################
|
30
|
+
job_script =<<"BODY"
|
31
|
+
#!/bin/bash
|
32
|
+
LANG=C
|
33
|
+
mkdir -p #{job_info[:job_script_id]}
|
34
|
+
cd #{job_info[:job_script_id]}
|
35
|
+
echo \"#---------- header of #{job_info[:job_id]} ----------\" > nera_status.rb
|
36
|
+
echo \"header = {\" >> nera_status.rb
|
37
|
+
echo \" :database => '#{job_info[:database]}',\" >> nera_status.rb
|
38
|
+
echo \" :job_id => #{job_info[:job_id]},\" >> nera_status.rb
|
39
|
+
echo \" :simulator => '#{job_info[:simulator]}',\" >> nera_status.rb
|
40
|
+
echo \" :simulator_id => #{job_info[:sim_id]},\" >> nera_status.rb
|
41
|
+
echo \" :param_id => #{job_info[:param_id]},\" >> nera_status.rb
|
42
|
+
echo \" :run_ids => #{job_info[:run_ids].inspect},\" >> nera_status.rb
|
43
|
+
echo \" :parameters => {\" >> nera_status.rb
|
44
|
+
BODY
|
45
|
+
|
46
|
+
job_info[:params].each_pair do |key, value|
|
47
|
+
job_script += "echo \" :#{key} => #{value},\" >> nera_status.rb\n"
|
48
|
+
end
|
49
|
+
|
50
|
+
job_script +=<<"BODY"
|
51
|
+
echo \" },\" >> nera_status.rb
|
52
|
+
echo \" :start_at => DateTime.parse( <<'NERA_DATE'.chomp )\" >> nera_status.rb
|
53
|
+
date >> nera_status.rb
|
54
|
+
echo \"NERA_DATE\" >> nera_status.rb
|
55
|
+
echo \"}\" >> nera_status.rb
|
56
|
+
echo \"body = {\" >> nera_status.rb
|
57
|
+
BODY
|
58
|
+
|
59
|
+
job_info[:run_ids].each_with_index do |run_id, i|
|
60
|
+
job_script +=<<"BODY"
|
61
|
+
echo \" #---------- #{run_id} start ----------\" >> nera_status.rb
|
62
|
+
echo \" #{run_id} => {\" >> nera_status.rb
|
63
|
+
echo \" :seed => #{job_info[:seeds][i]}, :start_at => DateTime.parse( <<'NERA_START_AT'.chomp), :output => <<'NERA_OUTPUT', :finish_at => DateTime.parse( <<'NERA_FINISH_AT'.chomp)\" >> nera_status.rb
|
64
|
+
date >> nera_status.rb
|
65
|
+
echo \"NERA_START_AT\" >> nera_status.rb
|
66
|
+
echo \"#{job_info[:execs][i]}\"
|
67
|
+
mkdir -p #{run_id}
|
68
|
+
cd #{run_id}
|
69
|
+
{ time -p #{job_info[:execs][i]}; } >> ../nera_status.rb 2>&1
|
70
|
+
if test $? -ne 0; then { echo \"Exit abnormally!\" >> ../nera_status.rb; exit; } fi
|
71
|
+
cd ..
|
72
|
+
echo \"NERA_OUTPUT\" >> nera_status.rb
|
73
|
+
date >> nera_status.rb
|
74
|
+
echo \"NERA_FINISH_AT\" >> nera_status.rb
|
75
|
+
BODY
|
76
|
+
if i == job_info[:run_ids].length-1 then job_script += "echo \" }\" >> nera_status.rb\n"
|
77
|
+
else job_script += "echo \" },\" >> nera_status.rb\n" end
|
78
|
+
job_script += "echo \" #---------- #{run_id} end ----------\" >> nera_status.rb\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
job_script +=<<"BODY"
|
82
|
+
echo \"}\" >> nera_status.rb
|
83
|
+
echo \"footer = {\" >> nera_status.rb
|
84
|
+
echo \" :host_name => <<'NERA_HOST'.chomp, :finish_at => DateTime.parse( <<'NERA_DATE'.chomp )\" >> nera_status.rb
|
85
|
+
hostname >> nera_status.rb
|
86
|
+
echo \"NERA_HOST\" >> nera_status.rb
|
87
|
+
date >> nera_status.rb
|
88
|
+
echo \"NERA_DATE\" >> nera_status.rb
|
89
|
+
echo "}" >> nera_status.rb
|
90
|
+
cd ..
|
91
|
+
tar cf #{job_info[:job_script_id]}.tar #{job_info[:job_script_id]}/
|
92
|
+
if test $? -ne 0; then { echo \"#Exit abnormally!\" >> ./nera_status.rb; exit; } fi
|
93
|
+
bzip2 #{job_info[:job_script_id]}.tar
|
94
|
+
if test $? -ne 0; then { echo \"Exit abnormally!\"; exit; } fi
|
95
|
+
rm -rf #{job_info[:job_script_id]}
|
96
|
+
BODY
|
97
|
+
############## job script ###################
|
98
|
+
|
99
|
+
return job_script
|
100
|
+
end
|
101
|
+
private :job_script_body
|
102
|
+
|
103
|
+
#--------------------------------------
|
104
|
+
#-public instance methods--------------
|
105
|
+
#--------------------------------------
|
106
|
+
public
|
107
|
+
# create job script --------------------
|
108
|
+
def create_script(job_id, sim_id, param_id, sim_inst, run_stat)
|
109
|
+
unless job_id.is_a?(Integer) and sim_id.is_a?(Integer) and param_id.is_a?(Integer)
|
110
|
+
raise ArgumentError, "Each ID of Job, Simulation and Parameter must be an Integer."
|
111
|
+
end
|
112
|
+
unless sim_inst.kind_of?(NERA::Simulator)
|
113
|
+
raise ArgumentError, "Simulation_Instance must be a subclass of NERA::Simulator."
|
114
|
+
else
|
115
|
+
if sim_inst.param == {}
|
116
|
+
raise ArgumentError, "@param in Simulation_Instance must be set a parameter."
|
117
|
+
end
|
118
|
+
end
|
119
|
+
unless run_stat.is_a?(Array)
|
120
|
+
raise ArgumentError, "Run_Status must be an Array."
|
121
|
+
else
|
122
|
+
run_stat.each do | stat |
|
123
|
+
unless stat.has_key?(:id) and stat.has_key?(:seed)
|
124
|
+
raise ArgumentError, "Run Status must have keys, :id and :seed."
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
run_ids =[]
|
131
|
+
seeds = []
|
132
|
+
run_stat.each do |run_info|
|
133
|
+
run_ids << run_info[:id]
|
134
|
+
seeds << run_info[:seed]
|
135
|
+
end
|
136
|
+
|
137
|
+
job_info ={
|
138
|
+
:job_id => job_id,
|
139
|
+
:job_script_id => sprintf("j%06d",job_id),
|
140
|
+
:database => File.basename( @db_folder.path_to_simulator_layer),
|
141
|
+
:simulator => sim_inst.class.to_s,
|
142
|
+
:sim_id => sim_id,
|
143
|
+
:param_id => param_id,
|
144
|
+
:params => sim_inst.param,
|
145
|
+
# {:L => 256, :K => 0.22325, :tmax => 512},
|
146
|
+
:run_ids => run_ids,
|
147
|
+
:seeds => seeds,
|
148
|
+
:execs => seeds.map do |sed| sim_inst.simulate_command( sed) end
|
149
|
+
}
|
150
|
+
|
151
|
+
path_job_script = @db_folder.path_to_job_script( job_id)
|
152
|
+
unless FileTest.directory?(@db_folder.path_to_job_layer)
|
153
|
+
raise "must not happen"
|
154
|
+
end
|
155
|
+
if FileTest.exist?(path_job_script)
|
156
|
+
raise "must not happen"
|
157
|
+
end
|
158
|
+
|
159
|
+
File.open(path_job_script,"w") do |io|
|
160
|
+
io.puts job_script_body(job_info)
|
161
|
+
io.flush
|
162
|
+
end
|
163
|
+
|
164
|
+
return path_job_script
|
165
|
+
end
|
166
|
+
|
167
|
+
# parse status file --------------------
|
168
|
+
def parse_status(path_stat)
|
169
|
+
unless path_stat.is_a?(String) and FileTest.exists?(path_stat)
|
170
|
+
raise ArgumentError, "#{path_stat} is not valid status file."
|
171
|
+
end
|
172
|
+
|
173
|
+
str = ""
|
174
|
+
File.open(path_stat).each do |line|
|
175
|
+
str += line
|
176
|
+
end
|
177
|
+
header = {}
|
178
|
+
body = {}
|
179
|
+
footer = {}
|
180
|
+
begin
|
181
|
+
eval(str)
|
182
|
+
rescue
|
183
|
+
raise RuntimeError, "This nera_status.rb is not valid."
|
184
|
+
end
|
185
|
+
job_info = header.merge( footer)
|
186
|
+
|
187
|
+
body.each_pair do |run_id, info|
|
188
|
+
info[:real_time] = 0.0
|
189
|
+
info[:user_time] = 0.0
|
190
|
+
info[:output].each_line do |line|
|
191
|
+
if line =~ /^real [-+]?(?:[0-9]+(\.[0-9]*)?|(\.[0-9]+))([eE][-+]?[0-9]+)?/
|
192
|
+
info[:real_time] = line.sub(/^real /,'').to_f
|
193
|
+
elsif line =~ /^user [-+]?(?:[0-9]+(\.[0-9]*)?|(\.[0-9]+))([eE][-+]?[0-9]+)?/
|
194
|
+
info[:user_time] += line.sub(/^user /,'').to_f
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
return job_info, body
|
199
|
+
end
|
200
|
+
|
201
|
+
end
|
202
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'nera_db_folders'
|
3
|
+
require 'nera_simulator_records'
|
4
|
+
require 'nera_parameter_records'
|
5
|
+
require 'nera_run_records'
|
6
|
+
require 'nera_job_records'
|
7
|
+
|
8
|
+
module NERA
|
9
|
+
|
10
|
+
class ParameterLayerController
|
11
|
+
|
12
|
+
# instance of NERA::DbFolders
|
13
|
+
@db_folders
|
14
|
+
# simulator class
|
15
|
+
@sim_class
|
16
|
+
# instance of NERA::ParameterRecords
|
17
|
+
@param_records
|
18
|
+
|
19
|
+
def initialize( path_db_folder, sim_class)
|
20
|
+
raise ArgumentError unless path_db_folder.is_a?(String)
|
21
|
+
unless sim_class.is_a?(Class) and sim_class.superclass == NERA::Simulator
|
22
|
+
raise ArgumentError, "#{sim_class} is not a subclass of NERA::Simulator"
|
23
|
+
end
|
24
|
+
|
25
|
+
@db_folders = NERA::DbFolders.new( path_db_folder)
|
26
|
+
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
27
|
+
@sim_class = sim_class
|
28
|
+
p_tab_path = @db_folders.path_to_parameters_table( @sim_class)
|
29
|
+
@param_records = NERA::ParameterRecords.new( p_tab_path, @sim_class)
|
30
|
+
end
|
31
|
+
|
32
|
+
def parameters_list_in_csv
|
33
|
+
l_active = @param_records.list_active
|
34
|
+
header = @param_records.keys.dup
|
35
|
+
header.delete( :in_trashbox?)
|
36
|
+
csv_list = []
|
37
|
+
l_active.each do |p_hash|
|
38
|
+
strings = []
|
39
|
+
header.each do |key|
|
40
|
+
if p_hash[key].is_a?(DateTime)
|
41
|
+
strings << p_hash[key].to_s.split('T')[0]
|
42
|
+
elsif p_hash[key]
|
43
|
+
strings << p_hash[key].to_s
|
44
|
+
elsif p_hash[key] == nil
|
45
|
+
found = @sim_class::Parameters.find do |p| p[0] == key end
|
46
|
+
strings << found[2].to_s
|
47
|
+
end
|
48
|
+
end
|
49
|
+
csv_list << strings.join(", ")
|
50
|
+
end
|
51
|
+
return header.join(", "), csv_list
|
52
|
+
end
|
53
|
+
|
54
|
+
def get_id_from_csv_string( pstr)
|
55
|
+
return pstr.split(',').first.to_i
|
56
|
+
end
|
57
|
+
|
58
|
+
def list_of_parameters
|
59
|
+
return @sim_class::Parameters.dup
|
60
|
+
end
|
61
|
+
|
62
|
+
def path_to_param_layer
|
63
|
+
return @db_folders.path_to_parameter_layer( @sim_class)
|
64
|
+
end
|
65
|
+
|
66
|
+
def create_a_new_parameter_set( p_hash)
|
67
|
+
pid = nil
|
68
|
+
@param_records.transaction {
|
69
|
+
pid = @param_records.add( p_hash)
|
70
|
+
return nil unless pid
|
71
|
+
b = @db_folders.create_new_parameter_folder( @sim_class, pid)
|
72
|
+
raise "Creation of a new folder for #{kalss}, #{pid} failed." unless b
|
73
|
+
path = @db_folders.path_to_runs_table( @sim_class, pid)
|
74
|
+
c = NERA::RunRecords.create_table( path)
|
75
|
+
raise "File #{path} already exists." unless c
|
76
|
+
}
|
77
|
+
dump_in_yaml
|
78
|
+
return pid
|
79
|
+
end
|
80
|
+
|
81
|
+
def move_a_parameter_set_into_trashbox( id)
|
82
|
+
return nil unless @param_records.find_by_id(id)
|
83
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id) )
|
84
|
+
return nil if run_records.list_all_not_finished.size > 0
|
85
|
+
|
86
|
+
job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
87
|
+
job_records.transaction {
|
88
|
+
@param_records.transaction {
|
89
|
+
flag = @param_records.update_to_state_trashbox(id)
|
90
|
+
return nil unless flag
|
91
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
92
|
+
cmd = "tar cvjf #{r_path}.tar.bz2 #{r_path}"
|
93
|
+
system(cmd)
|
94
|
+
raise "Command #{cmd} failed" unless $? == 0
|
95
|
+
FileUtils.rm_r(r_path)
|
96
|
+
}
|
97
|
+
}
|
98
|
+
dump_in_yaml
|
99
|
+
return true
|
100
|
+
end
|
101
|
+
|
102
|
+
def trashbox_parameter_list_in_csv
|
103
|
+
lt = @param_records.list_trashbox
|
104
|
+
header = @param_records.keys.dup
|
105
|
+
header.delete( :in_trashbox?)
|
106
|
+
csv_list = []
|
107
|
+
lt.each do |p_hash|
|
108
|
+
strings = []
|
109
|
+
header.each do |key|
|
110
|
+
if p_hash[key].is_a?(DateTime)
|
111
|
+
strings << p_hash[key].to_s.split('T')[0]
|
112
|
+
elsif p_hash[key]
|
113
|
+
strings << p_hash[key].to_s
|
114
|
+
elsif p_hash[key] == nil
|
115
|
+
found = @sim_class::Parameters.find do |p| p[0] == key end
|
116
|
+
strings << found[2].to_s
|
117
|
+
end
|
118
|
+
end
|
119
|
+
csv_list << strings.join(", ")
|
120
|
+
end
|
121
|
+
return header.join(", "), csv_list
|
122
|
+
end
|
123
|
+
|
124
|
+
def revert_a_parameter_set_in_trashbox( id)
|
125
|
+
@param_records.transaction {
|
126
|
+
flag = @param_records.update_to_state_active(id)
|
127
|
+
return nil unless flag
|
128
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
129
|
+
cmd = "tar xvjf #{r_path}.tar.bz2"
|
130
|
+
system(cmd)
|
131
|
+
raise "Command #{cmd} failed" unless $? == 0
|
132
|
+
FileUtils.rm(r_path+".tar.bz2")
|
133
|
+
}
|
134
|
+
dump_in_yaml
|
135
|
+
return true
|
136
|
+
end
|
137
|
+
|
138
|
+
def delete_a_parameter_set( id)
|
139
|
+
@param_records.transaction {
|
140
|
+
flag = @param_records.destroy(id)
|
141
|
+
return nil unless flag
|
142
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
143
|
+
FileUtils.rm("#{r_path}.tar.bz2")
|
144
|
+
}
|
145
|
+
dump_in_yaml
|
146
|
+
return true
|
147
|
+
end
|
148
|
+
|
149
|
+
def dump_in_yaml
|
150
|
+
File.open( @db_folders.path_to_parameters_yaml( @sim_class), 'w') do |io|
|
151
|
+
YAML.dump( @param_records.list_all, io)
|
152
|
+
io.flush
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
end
|