nera 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/Manifest.txt +37 -0
- data/README.rdoc +47 -0
- data/Rakefile +27 -0
- data/bin/nera +29 -0
- data/bin/nera_addsim +30 -0
- data/lib/nera/nera_cui.rb +417 -0
- data/lib/nera/nera_database.rb +281 -0
- data/lib/nera/nera_db_folders.rb +226 -0
- data/lib/nera/nera_dialog.rb +205 -0
- data/lib/nera/nera_job_layer_controller.rb +237 -0
- data/lib/nera/nera_job_records.rb +111 -0
- data/lib/nera/nera_job_script.rb +202 -0
- data/lib/nera/nera_parameter_layer_controller.rb +157 -0
- data/lib/nera/nera_parameter_records.rb +186 -0
- data/lib/nera/nera_run_layer_controller.rb +192 -0
- data/lib/nera/nera_run_records.rb +184 -0
- data/lib/nera/nera_simulator.rb +26 -0
- data/lib/nera/nera_simulator_layer_controller.rb +66 -0
- data/lib/nera/nera_simulator_records.rb +84 -0
- data/lib/nera.rb +25 -0
- data/lib/nera_addsim/make_simulator.rb +307 -0
- data/scripts/make_manifest.rb +21 -0
- data/test/runner.rb +3 -0
- data/test/test_helper.rb +52 -0
- data/test/test_nera_database.rb +221 -0
- data/test/test_nera_db_folders.rb +209 -0
- data/test/test_nera_dialog.rb +134 -0
- data/test/test_nera_job_layer_controller.rb +132 -0
- data/test/test_nera_job_records.rb +260 -0
- data/test/test_nera_parameter_layer_controller.rb +188 -0
- data/test/test_nera_parameter_records.rb +285 -0
- data/test/test_nera_run_layer_controller.rb +171 -0
- data/test/test_nera_run_records.rb +290 -0
- data/test/test_nera_simulator.rb +26 -0
- data/test/test_nera_simulator_layer_controller.rb +54 -0
- data/test/test_nera_simulator_records.rb +140 -0
- metadata +125 -0
@@ -0,0 +1,237 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'yaml'
|
4
|
+
require 'nera_db_folders'
|
5
|
+
require 'nera_simulator_records'
|
6
|
+
require 'nera_parameter_records'
|
7
|
+
require 'nera_run_records'
|
8
|
+
require 'nera_job_records'
|
9
|
+
require 'nera_job_script'
|
10
|
+
|
11
|
+
module NERA
|
12
|
+
|
13
|
+
class JobLayerController
|
14
|
+
|
15
|
+
# instance of NERA::DbFolders
|
16
|
+
@db_folders
|
17
|
+
# instance of NERA::JobRecords
|
18
|
+
@job_records
|
19
|
+
|
20
|
+
def initialize( path_db_folder)
|
21
|
+
raise ArgumentError unless path_db_folder.is_a?(String)
|
22
|
+
|
23
|
+
@db_folders = NERA::DbFolders.new( path_db_folder)
|
24
|
+
@job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
25
|
+
end
|
26
|
+
|
27
|
+
public
|
28
|
+
def path_to_job_layer
|
29
|
+
return @db_folders.path_to_job_layer
|
30
|
+
end
|
31
|
+
|
32
|
+
def not_finished_list_in_csv
|
33
|
+
list = @job_records.list_all
|
34
|
+
header = @job_records.keys.dup
|
35
|
+
header_ab = "id, state, simulator, param_id, run_id, num_runs, created_at, updated_at, host_name"
|
36
|
+
|
37
|
+
csv_list = []
|
38
|
+
list.each do |r_hash|
|
39
|
+
strings = []
|
40
|
+
header.each do |key|
|
41
|
+
if r_hash[key].is_a?(DateTime)
|
42
|
+
strings << r_hash[key].to_s.split('T')[0]
|
43
|
+
else
|
44
|
+
strings << r_hash[key].to_s
|
45
|
+
end
|
46
|
+
end
|
47
|
+
csv_list << strings.join(", ")
|
48
|
+
end
|
49
|
+
return header_ab, csv_list
|
50
|
+
end
|
51
|
+
|
52
|
+
def cancel_jobs( job_ids)
|
53
|
+
unless job_ids.is_a?(Array)
|
54
|
+
raise ArgumentError, "job_ids must be an Array."
|
55
|
+
end
|
56
|
+
job_ids.each do |x|
|
57
|
+
raise ArgumentError, "each element of job_ids must be an Integer" unless x.is_a?(Integer)
|
58
|
+
end
|
59
|
+
|
60
|
+
flag = nil
|
61
|
+
@job_records.transaction {
|
62
|
+
job_ids.each do |jid|
|
63
|
+
d = @job_records.destroy( jid)
|
64
|
+
next unless d
|
65
|
+
sim_recs = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
66
|
+
klass = eval( d[:simulator])
|
67
|
+
raise RuntimeError, "must not happen" unless klass
|
68
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( klass, d[:parameter_id]) )
|
69
|
+
run_records.transaction {
|
70
|
+
a = run_records.destroy_job_id( jid)
|
71
|
+
raise RuntimeError, "must not happen" unless a
|
72
|
+
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
73
|
+
}
|
74
|
+
File.open( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]), 'w') do |io|
|
75
|
+
YAML.dump( run_records.list_all, io)
|
76
|
+
io.flush
|
77
|
+
end
|
78
|
+
flag = true
|
79
|
+
end
|
80
|
+
}
|
81
|
+
dump_in_yaml
|
82
|
+
return flag
|
83
|
+
end
|
84
|
+
|
85
|
+
def include_list
|
86
|
+
list = @db_folders.search_include_files
|
87
|
+
list.map! do |path|
|
88
|
+
path.sub( @db_folders.path_to_include_layer, '')
|
89
|
+
end
|
90
|
+
return list
|
91
|
+
end
|
92
|
+
|
93
|
+
def include( filename)
|
94
|
+
@job_records.transaction {
|
95
|
+
fpath = @db_folders.path_to_include_layer + filename
|
96
|
+
return nil unless File.exist?(fpath)
|
97
|
+
jinfo, rinfo = expand_and_parse( fpath)
|
98
|
+
run_layer_path, run_recs, run_yaml_path = check_consistency( jinfo, rinfo)
|
99
|
+
run_recs.transaction {
|
100
|
+
folder_path = fpath.sub(/.tar.bz2$/,'/')
|
101
|
+
file_move( folder_path, run_layer_path, jinfo[:run_ids])
|
102
|
+
update_tables( run_recs, jinfo, rinfo)
|
103
|
+
}
|
104
|
+
remove_files( fpath)
|
105
|
+
File.open( run_yaml_path, 'w') do |io|
|
106
|
+
YAML.dump( run_recs.list_all, io)
|
107
|
+
io.flush
|
108
|
+
end
|
109
|
+
}
|
110
|
+
dump_in_yaml
|
111
|
+
return true
|
112
|
+
end
|
113
|
+
|
114
|
+
private
|
115
|
+
def expand_and_parse( fpath)
|
116
|
+
bz2_path = fpath.sub( /^#{@db_folders.path_to_include_layer}/, '')
|
117
|
+
Dir.chdir( @db_folders.path_to_include_layer) {
|
118
|
+
cmd = "bunzip2 #{bz2_path}"
|
119
|
+
system(cmd)
|
120
|
+
raise "command : #{cmd} failed" unless $? == 0
|
121
|
+
tarpath = bz2_path.sub(/.bz2$/,'')
|
122
|
+
cmd = "tar xvf #{tarpath}"
|
123
|
+
system(cmd)
|
124
|
+
raise "command : #{cmd} failed" unless $? == 0
|
125
|
+
stat_path = tarpath.sub(/.tar$/,'/nera_status.rb')
|
126
|
+
js = NERA::JobScript.new(@db_folders)
|
127
|
+
jinfo, rinfo = js.parse_status( stat_path)
|
128
|
+
return jinfo, rinfo
|
129
|
+
}
|
130
|
+
end
|
131
|
+
|
132
|
+
private
|
133
|
+
def check_consistency( jinfo, rinfo)
|
134
|
+
# record exists?
|
135
|
+
found = @job_records.find_by_id( jinfo[:job_id] )
|
136
|
+
return nil unless found
|
137
|
+
|
138
|
+
# check consistency of the simulator
|
139
|
+
conflicting unless jinfo[:simulator] == found[:simulator]
|
140
|
+
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
141
|
+
sim_class = eval( jinfo[:simulator])
|
142
|
+
conflicting unless jinfo[:simulator] == sim_class.to_s
|
143
|
+
|
144
|
+
# check consistency of the parameter
|
145
|
+
conflicting unless jinfo[:param_id] == found[:parameter_id]
|
146
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class) , sim_class)
|
147
|
+
param = param_records.find_by_id( jinfo[:param_id])
|
148
|
+
jinfo[:parameters].each_pair do |key, value|
|
149
|
+
conflicting unless param[key] == value
|
150
|
+
end
|
151
|
+
|
152
|
+
# check consistency of the runs
|
153
|
+
conflicting unless jinfo[:run_ids][0] == found[:run_id]
|
154
|
+
conflicting unless jinfo[:run_ids].size == found[:number_of_runs]
|
155
|
+
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]) )
|
156
|
+
run_yaml_path = @db_folders.path_to_runs_yaml( sim_class, jinfo[:param_id])
|
157
|
+
found_runs = run_recs.list_all_not_finished.find_all do |rec|
|
158
|
+
rec[:job_id] == jinfo[:job_id] and jinfo[:run_ids].include?(rec[:id])
|
159
|
+
end
|
160
|
+
conflicting unless jinfo[:run_ids].size == found_runs.size
|
161
|
+
|
162
|
+
# check consistency of run_info
|
163
|
+
raise "Run info is not consistent with job info!" unless jinfo[:run_ids] == rinfo.keys.sort
|
164
|
+
|
165
|
+
run_layer_path = @db_folders.path_to_run_layer( sim_class, jinfo[:param_id])
|
166
|
+
return run_layer_path, run_recs, run_yaml_path
|
167
|
+
end
|
168
|
+
|
169
|
+
def conflicting
|
170
|
+
raise RuntimeError, "This information is conflicting with the table."
|
171
|
+
end
|
172
|
+
|
173
|
+
def file_move( folder_path, run_layer_path, run_ids)
|
174
|
+
run_ids.each do |rid|
|
175
|
+
data_folder = folder_path + rid.to_s
|
176
|
+
raise "must not happen" unless FileTest.directory?(data_folder)
|
177
|
+
data_files = Dir.glob(data_folder + '/*')
|
178
|
+
data_files.each do |data_file|
|
179
|
+
dest = run_layer_path + sprintf("%06d_",rid) + File.basename( data_file)
|
180
|
+
raise "file already exists." if File.exists?(dest)
|
181
|
+
FileUtils.mv( data_file, dest, {:verbose => true} )
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
def update_tables( run_recs, jinfo, rinfos)
|
187
|
+
d = DateTime.now
|
188
|
+
rinfos.each_pair do |rid, info|
|
189
|
+
f = run_recs.update_state_to_finished( rid, info[:start_at], info[:finish_at], d,
|
190
|
+
info[:real_time], info[:user_time], jinfo[:host_name] )
|
191
|
+
raise "must not happen" unless f
|
192
|
+
end
|
193
|
+
destroyed = @job_records.destroy( jinfo[:job_id])
|
194
|
+
destroyed[:simulator] = jinfo[:simulator]
|
195
|
+
destroyed[:parameters] = jinfo[:parameters]
|
196
|
+
destroyed[:start_at] = jinfo[:start_at]
|
197
|
+
destroyed[:host_name] = jinfo[:host_name]
|
198
|
+
destroyed[:finish_at] = jinfo[:finish_at]
|
199
|
+
File.open( @db_folders.path_to_finished_jobs_file, "a") do |io|
|
200
|
+
YAML.dump( destroyed, io)
|
201
|
+
io.flush
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def remove_files( fpath)
|
206
|
+
tar_path = fpath.sub(/.bz2$/,'')
|
207
|
+
FileUtils.rm( tar_path)
|
208
|
+
fold_path = tar_path.sub(/.tar$/,'/')
|
209
|
+
FileUtils.rm_r( fold_path)
|
210
|
+
sh_path = @db_folders.path_to_job_layer + File.basename( tar_path.sub(/.tar$/,'.sh'))
|
211
|
+
if FileTest.exist?(sh_path)
|
212
|
+
ff = @db_folders.path_to_job_layer + 'finished_jobs/'
|
213
|
+
FileUtils.mkdir_p( ff)
|
214
|
+
FileUtils.mv(sh_path, ff )
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
public
|
219
|
+
def include_all
|
220
|
+
flag = true
|
221
|
+
include_list.each do |fname|
|
222
|
+
f = include(fname)
|
223
|
+
flag = nil unless f
|
224
|
+
end
|
225
|
+
dump_in_yaml
|
226
|
+
return flag
|
227
|
+
end
|
228
|
+
|
229
|
+
def dump_in_yaml
|
230
|
+
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
231
|
+
YAML.dump( @job_records.list_all, io)
|
232
|
+
io.flush
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
end
|
237
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require 'nera_database'
|
2
|
+
|
3
|
+
module NERA
|
4
|
+
|
5
|
+
# This class access the simulator table.
|
6
|
+
class JobRecords
|
7
|
+
|
8
|
+
# keys of the simulator table
|
9
|
+
ATTRIBUTES = [ [:id, Integer],
|
10
|
+
[:state, Symbol],
|
11
|
+
[:simulator, String],
|
12
|
+
[:parameter_id, Integer],
|
13
|
+
[:run_id, Integer],
|
14
|
+
[:number_of_runs, Integer],
|
15
|
+
[:created_at, DateTime],
|
16
|
+
[:updated_at, DateTime],
|
17
|
+
[:host_name, String]
|
18
|
+
]
|
19
|
+
|
20
|
+
# NERA::Database object
|
21
|
+
@db
|
22
|
+
|
23
|
+
# db_file is the path to database file
|
24
|
+
def initialize( db_file)
|
25
|
+
@db = NERA::Database.new( db_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
# if file already exists, returns false
|
29
|
+
def self.create_table( db_file)
|
30
|
+
NERA::Database.create_table( db_file)
|
31
|
+
end
|
32
|
+
|
33
|
+
# return keys of the simulator table
|
34
|
+
def keys
|
35
|
+
keys = ATTRIBUTES.map do |x|
|
36
|
+
x[0]
|
37
|
+
end
|
38
|
+
return keys
|
39
|
+
end
|
40
|
+
|
41
|
+
# sim_class must be a subclass of NERA::Simulator
|
42
|
+
def add( sim_class, param_id, r_id, num_runs)
|
43
|
+
unless sim_class.is_a?(Class) and sim_class.superclass == NERA::Simulator and param_id.is_a?(Integer) and r_id.is_a?(Integer) and num_runs.is_a?(Integer)
|
44
|
+
raise ArgumentError, "Each argument must be an Integer"
|
45
|
+
end
|
46
|
+
if param_id <= 0 or r_id <= 0 or num_runs <= 0
|
47
|
+
raise ArgumentError, "Each argument must be a positive integer."
|
48
|
+
end
|
49
|
+
d = DateTime.now
|
50
|
+
h = { :state => :created, :simulator => sim_class.to_s, :parameter_id => param_id, :run_id => r_id,
|
51
|
+
:number_of_runs => num_runs, :created_at => d, :updated_at => d, :host_name => nil}
|
52
|
+
id = @db.add( h)
|
53
|
+
return id
|
54
|
+
end
|
55
|
+
|
56
|
+
def list_all
|
57
|
+
l = @db.find_all do |r| true end
|
58
|
+
return l.to_a
|
59
|
+
end
|
60
|
+
|
61
|
+
def list_created
|
62
|
+
l = @db.find_all do |r|
|
63
|
+
r[:state] == :created
|
64
|
+
end
|
65
|
+
return l.to_a
|
66
|
+
end
|
67
|
+
|
68
|
+
def find_by_id( i)
|
69
|
+
@db.find_by_id( i)
|
70
|
+
end
|
71
|
+
|
72
|
+
def update_to_state_copied( id, hostname)
|
73
|
+
unless id.is_a?(Integer) and hostname.is_a?(String)
|
74
|
+
raise ArgumentError, "Type of the argument is invalid."
|
75
|
+
end
|
76
|
+
|
77
|
+
found = @db.find_by_id( id)
|
78
|
+
return nil if found == nil
|
79
|
+
|
80
|
+
found[:state] = :copied
|
81
|
+
found[:updated_at] = DateTime.now
|
82
|
+
found[:host_name] = hostname
|
83
|
+
@db.update(found)
|
84
|
+
end
|
85
|
+
|
86
|
+
def update_to_state_submitted( id, hostname)
|
87
|
+
unless id.is_a?(Integer) and hostname.is_a?(String)
|
88
|
+
raise ArgumentError, "Type of the argument is invalid."
|
89
|
+
end
|
90
|
+
|
91
|
+
found = @db.find_by_id( id)
|
92
|
+
return nil if found == nil
|
93
|
+
|
94
|
+
found[:state] = :submitted
|
95
|
+
found[:updated_at] = DateTime.now
|
96
|
+
found[:host_name] = hostname
|
97
|
+
@db.update(found)
|
98
|
+
end
|
99
|
+
|
100
|
+
def destroy( id)
|
101
|
+
raise ArgumentError unless id.is_a?(Integer)
|
102
|
+
@db.destroy(id)
|
103
|
+
end
|
104
|
+
|
105
|
+
def transaction
|
106
|
+
@db.transaction{
|
107
|
+
yield
|
108
|
+
}
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,202 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'date'
|
3
|
+
require 'nera_db_folders'
|
4
|
+
require 'nera_simulator'
|
5
|
+
|
6
|
+
|
7
|
+
module NERA
|
8
|
+
|
9
|
+
# This class create & parse job scripts
|
10
|
+
class JobScript
|
11
|
+
|
12
|
+
# NERA::DbFolders object
|
13
|
+
@db_folder
|
14
|
+
|
15
|
+
#--------------------------------------
|
16
|
+
#-class methods -----------------------
|
17
|
+
#--------------------------------------
|
18
|
+
|
19
|
+
# db_folder is the instance of DbFolders
|
20
|
+
def initialize( db_folder)
|
21
|
+
unless db_folder.instance_of?(NERA::DbFolders) then
|
22
|
+
raise ArgumentError, "Argument must be an instance of NERA::DbFolders."
|
23
|
+
end
|
24
|
+
@db_folder = db_folder
|
25
|
+
end
|
26
|
+
|
27
|
+
# job script
|
28
|
+
def job_script_body(job_info)
|
29
|
+
############## job_script ###################
|
30
|
+
job_script =<<"BODY"
|
31
|
+
#!/bin/bash
|
32
|
+
LANG=C
|
33
|
+
mkdir -p #{job_info[:job_script_id]}
|
34
|
+
cd #{job_info[:job_script_id]}
|
35
|
+
echo \"#---------- header of #{job_info[:job_id]} ----------\" > nera_status.rb
|
36
|
+
echo \"header = {\" >> nera_status.rb
|
37
|
+
echo \" :database => '#{job_info[:database]}',\" >> nera_status.rb
|
38
|
+
echo \" :job_id => #{job_info[:job_id]},\" >> nera_status.rb
|
39
|
+
echo \" :simulator => '#{job_info[:simulator]}',\" >> nera_status.rb
|
40
|
+
echo \" :simulator_id => #{job_info[:sim_id]},\" >> nera_status.rb
|
41
|
+
echo \" :param_id => #{job_info[:param_id]},\" >> nera_status.rb
|
42
|
+
echo \" :run_ids => #{job_info[:run_ids].inspect},\" >> nera_status.rb
|
43
|
+
echo \" :parameters => {\" >> nera_status.rb
|
44
|
+
BODY
|
45
|
+
|
46
|
+
job_info[:params].each_pair do |key, value|
|
47
|
+
job_script += "echo \" :#{key} => #{value},\" >> nera_status.rb\n"
|
48
|
+
end
|
49
|
+
|
50
|
+
job_script +=<<"BODY"
|
51
|
+
echo \" },\" >> nera_status.rb
|
52
|
+
echo \" :start_at => DateTime.parse( <<'NERA_DATE'.chomp )\" >> nera_status.rb
|
53
|
+
date >> nera_status.rb
|
54
|
+
echo \"NERA_DATE\" >> nera_status.rb
|
55
|
+
echo \"}\" >> nera_status.rb
|
56
|
+
echo \"body = {\" >> nera_status.rb
|
57
|
+
BODY
|
58
|
+
|
59
|
+
job_info[:run_ids].each_with_index do |run_id, i|
|
60
|
+
job_script +=<<"BODY"
|
61
|
+
echo \" #---------- #{run_id} start ----------\" >> nera_status.rb
|
62
|
+
echo \" #{run_id} => {\" >> nera_status.rb
|
63
|
+
echo \" :seed => #{job_info[:seeds][i]}, :start_at => DateTime.parse( <<'NERA_START_AT'.chomp), :output => <<'NERA_OUTPUT', :finish_at => DateTime.parse( <<'NERA_FINISH_AT'.chomp)\" >> nera_status.rb
|
64
|
+
date >> nera_status.rb
|
65
|
+
echo \"NERA_START_AT\" >> nera_status.rb
|
66
|
+
echo \"#{job_info[:execs][i]}\"
|
67
|
+
mkdir -p #{run_id}
|
68
|
+
cd #{run_id}
|
69
|
+
{ time -p #{job_info[:execs][i]}; } >> ../nera_status.rb 2>&1
|
70
|
+
if test $? -ne 0; then { echo \"Exit abnormally!\" >> ../nera_status.rb; exit; } fi
|
71
|
+
cd ..
|
72
|
+
echo \"NERA_OUTPUT\" >> nera_status.rb
|
73
|
+
date >> nera_status.rb
|
74
|
+
echo \"NERA_FINISH_AT\" >> nera_status.rb
|
75
|
+
BODY
|
76
|
+
if i == job_info[:run_ids].length-1 then job_script += "echo \" }\" >> nera_status.rb\n"
|
77
|
+
else job_script += "echo \" },\" >> nera_status.rb\n" end
|
78
|
+
job_script += "echo \" #---------- #{run_id} end ----------\" >> nera_status.rb\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
job_script +=<<"BODY"
|
82
|
+
echo \"}\" >> nera_status.rb
|
83
|
+
echo \"footer = {\" >> nera_status.rb
|
84
|
+
echo \" :host_name => <<'NERA_HOST'.chomp, :finish_at => DateTime.parse( <<'NERA_DATE'.chomp )\" >> nera_status.rb
|
85
|
+
hostname >> nera_status.rb
|
86
|
+
echo \"NERA_HOST\" >> nera_status.rb
|
87
|
+
date >> nera_status.rb
|
88
|
+
echo \"NERA_DATE\" >> nera_status.rb
|
89
|
+
echo "}" >> nera_status.rb
|
90
|
+
cd ..
|
91
|
+
tar cf #{job_info[:job_script_id]}.tar #{job_info[:job_script_id]}/
|
92
|
+
if test $? -ne 0; then { echo \"#Exit abnormally!\" >> ./nera_status.rb; exit; } fi
|
93
|
+
bzip2 #{job_info[:job_script_id]}.tar
|
94
|
+
if test $? -ne 0; then { echo \"Exit abnormally!\"; exit; } fi
|
95
|
+
rm -rf #{job_info[:job_script_id]}
|
96
|
+
BODY
|
97
|
+
############## job script ###################
|
98
|
+
|
99
|
+
return job_script
|
100
|
+
end
|
101
|
+
private :job_script_body
|
102
|
+
|
103
|
+
#--------------------------------------
|
104
|
+
#-public instance methods--------------
|
105
|
+
#--------------------------------------
|
106
|
+
public
|
107
|
+
# create job script --------------------
|
108
|
+
def create_script(job_id, sim_id, param_id, sim_inst, run_stat)
|
109
|
+
unless job_id.is_a?(Integer) and sim_id.is_a?(Integer) and param_id.is_a?(Integer)
|
110
|
+
raise ArgumentError, "Each ID of Job, Simulation and Parameter must be an Integer."
|
111
|
+
end
|
112
|
+
unless sim_inst.kind_of?(NERA::Simulator)
|
113
|
+
raise ArgumentError, "Simulation_Instance must be a subclass of NERA::Simulator."
|
114
|
+
else
|
115
|
+
if sim_inst.param == {}
|
116
|
+
raise ArgumentError, "@param in Simulation_Instance must be set a parameter."
|
117
|
+
end
|
118
|
+
end
|
119
|
+
unless run_stat.is_a?(Array)
|
120
|
+
raise ArgumentError, "Run_Status must be an Array."
|
121
|
+
else
|
122
|
+
run_stat.each do | stat |
|
123
|
+
unless stat.has_key?(:id) and stat.has_key?(:seed)
|
124
|
+
raise ArgumentError, "Run Status must have keys, :id and :seed."
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
run_ids =[]
|
131
|
+
seeds = []
|
132
|
+
run_stat.each do |run_info|
|
133
|
+
run_ids << run_info[:id]
|
134
|
+
seeds << run_info[:seed]
|
135
|
+
end
|
136
|
+
|
137
|
+
job_info ={
|
138
|
+
:job_id => job_id,
|
139
|
+
:job_script_id => sprintf("j%06d",job_id),
|
140
|
+
:database => File.basename( @db_folder.path_to_simulator_layer),
|
141
|
+
:simulator => sim_inst.class.to_s,
|
142
|
+
:sim_id => sim_id,
|
143
|
+
:param_id => param_id,
|
144
|
+
:params => sim_inst.param,
|
145
|
+
# {:L => 256, :K => 0.22325, :tmax => 512},
|
146
|
+
:run_ids => run_ids,
|
147
|
+
:seeds => seeds,
|
148
|
+
:execs => seeds.map do |sed| sim_inst.simulate_command( sed) end
|
149
|
+
}
|
150
|
+
|
151
|
+
path_job_script = @db_folder.path_to_job_script( job_id)
|
152
|
+
unless FileTest.directory?(@db_folder.path_to_job_layer)
|
153
|
+
raise "must not happen"
|
154
|
+
end
|
155
|
+
if FileTest.exist?(path_job_script)
|
156
|
+
raise "must not happen"
|
157
|
+
end
|
158
|
+
|
159
|
+
File.open(path_job_script,"w") do |io|
|
160
|
+
io.puts job_script_body(job_info)
|
161
|
+
io.flush
|
162
|
+
end
|
163
|
+
|
164
|
+
return path_job_script
|
165
|
+
end
|
166
|
+
|
167
|
+
# parse status file --------------------
|
168
|
+
def parse_status(path_stat)
|
169
|
+
unless path_stat.is_a?(String) and FileTest.exists?(path_stat)
|
170
|
+
raise ArgumentError, "#{path_stat} is not valid status file."
|
171
|
+
end
|
172
|
+
|
173
|
+
str = ""
|
174
|
+
File.open(path_stat).each do |line|
|
175
|
+
str += line
|
176
|
+
end
|
177
|
+
header = {}
|
178
|
+
body = {}
|
179
|
+
footer = {}
|
180
|
+
begin
|
181
|
+
eval(str)
|
182
|
+
rescue
|
183
|
+
raise RuntimeError, "This nera_status.rb is not valid."
|
184
|
+
end
|
185
|
+
job_info = header.merge( footer)
|
186
|
+
|
187
|
+
body.each_pair do |run_id, info|
|
188
|
+
info[:real_time] = 0.0
|
189
|
+
info[:user_time] = 0.0
|
190
|
+
info[:output].each_line do |line|
|
191
|
+
if line =~ /^real [-+]?(?:[0-9]+(\.[0-9]*)?|(\.[0-9]+))([eE][-+]?[0-9]+)?/
|
192
|
+
info[:real_time] = line.sub(/^real /,'').to_f
|
193
|
+
elsif line =~ /^user [-+]?(?:[0-9]+(\.[0-9]*)?|(\.[0-9]+))([eE][-+]?[0-9]+)?/
|
194
|
+
info[:user_time] += line.sub(/^user /,'').to_f
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
return job_info, body
|
199
|
+
end
|
200
|
+
|
201
|
+
end
|
202
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
require 'date'
|
2
|
+
require 'nera_db_folders'
|
3
|
+
require 'nera_simulator_records'
|
4
|
+
require 'nera_parameter_records'
|
5
|
+
require 'nera_run_records'
|
6
|
+
require 'nera_job_records'
|
7
|
+
|
8
|
+
module NERA
|
9
|
+
|
10
|
+
class ParameterLayerController
|
11
|
+
|
12
|
+
# instance of NERA::DbFolders
|
13
|
+
@db_folders
|
14
|
+
# simulator class
|
15
|
+
@sim_class
|
16
|
+
# instance of NERA::ParameterRecords
|
17
|
+
@param_records
|
18
|
+
|
19
|
+
def initialize( path_db_folder, sim_class)
|
20
|
+
raise ArgumentError unless path_db_folder.is_a?(String)
|
21
|
+
unless sim_class.is_a?(Class) and sim_class.superclass == NERA::Simulator
|
22
|
+
raise ArgumentError, "#{sim_class} is not a subclass of NERA::Simulator"
|
23
|
+
end
|
24
|
+
|
25
|
+
@db_folders = NERA::DbFolders.new( path_db_folder)
|
26
|
+
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
27
|
+
@sim_class = sim_class
|
28
|
+
p_tab_path = @db_folders.path_to_parameters_table( @sim_class)
|
29
|
+
@param_records = NERA::ParameterRecords.new( p_tab_path, @sim_class)
|
30
|
+
end
|
31
|
+
|
32
|
+
def parameters_list_in_csv
|
33
|
+
l_active = @param_records.list_active
|
34
|
+
header = @param_records.keys.dup
|
35
|
+
header.delete( :in_trashbox?)
|
36
|
+
csv_list = []
|
37
|
+
l_active.each do |p_hash|
|
38
|
+
strings = []
|
39
|
+
header.each do |key|
|
40
|
+
if p_hash[key].is_a?(DateTime)
|
41
|
+
strings << p_hash[key].to_s.split('T')[0]
|
42
|
+
elsif p_hash[key]
|
43
|
+
strings << p_hash[key].to_s
|
44
|
+
elsif p_hash[key] == nil
|
45
|
+
found = @sim_class::Parameters.find do |p| p[0] == key end
|
46
|
+
strings << found[2].to_s
|
47
|
+
end
|
48
|
+
end
|
49
|
+
csv_list << strings.join(", ")
|
50
|
+
end
|
51
|
+
return header.join(", "), csv_list
|
52
|
+
end
|
53
|
+
|
54
|
+
def get_id_from_csv_string( pstr)
|
55
|
+
return pstr.split(',').first.to_i
|
56
|
+
end
|
57
|
+
|
58
|
+
def list_of_parameters
|
59
|
+
return @sim_class::Parameters.dup
|
60
|
+
end
|
61
|
+
|
62
|
+
def path_to_param_layer
|
63
|
+
return @db_folders.path_to_parameter_layer( @sim_class)
|
64
|
+
end
|
65
|
+
|
66
|
+
def create_a_new_parameter_set( p_hash)
|
67
|
+
pid = nil
|
68
|
+
@param_records.transaction {
|
69
|
+
pid = @param_records.add( p_hash)
|
70
|
+
return nil unless pid
|
71
|
+
b = @db_folders.create_new_parameter_folder( @sim_class, pid)
|
72
|
+
raise "Creation of a new folder for #{kalss}, #{pid} failed." unless b
|
73
|
+
path = @db_folders.path_to_runs_table( @sim_class, pid)
|
74
|
+
c = NERA::RunRecords.create_table( path)
|
75
|
+
raise "File #{path} already exists." unless c
|
76
|
+
}
|
77
|
+
dump_in_yaml
|
78
|
+
return pid
|
79
|
+
end
|
80
|
+
|
81
|
+
def move_a_parameter_set_into_trashbox( id)
|
82
|
+
return nil unless @param_records.find_by_id(id)
|
83
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id) )
|
84
|
+
return nil if run_records.list_all_not_finished.size > 0
|
85
|
+
|
86
|
+
job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
87
|
+
job_records.transaction {
|
88
|
+
@param_records.transaction {
|
89
|
+
flag = @param_records.update_to_state_trashbox(id)
|
90
|
+
return nil unless flag
|
91
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
92
|
+
cmd = "tar cvjf #{r_path}.tar.bz2 #{r_path}"
|
93
|
+
system(cmd)
|
94
|
+
raise "Command #{cmd} failed" unless $? == 0
|
95
|
+
FileUtils.rm_r(r_path)
|
96
|
+
}
|
97
|
+
}
|
98
|
+
dump_in_yaml
|
99
|
+
return true
|
100
|
+
end
|
101
|
+
|
102
|
+
def trashbox_parameter_list_in_csv
|
103
|
+
lt = @param_records.list_trashbox
|
104
|
+
header = @param_records.keys.dup
|
105
|
+
header.delete( :in_trashbox?)
|
106
|
+
csv_list = []
|
107
|
+
lt.each do |p_hash|
|
108
|
+
strings = []
|
109
|
+
header.each do |key|
|
110
|
+
if p_hash[key].is_a?(DateTime)
|
111
|
+
strings << p_hash[key].to_s.split('T')[0]
|
112
|
+
elsif p_hash[key]
|
113
|
+
strings << p_hash[key].to_s
|
114
|
+
elsif p_hash[key] == nil
|
115
|
+
found = @sim_class::Parameters.find do |p| p[0] == key end
|
116
|
+
strings << found[2].to_s
|
117
|
+
end
|
118
|
+
end
|
119
|
+
csv_list << strings.join(", ")
|
120
|
+
end
|
121
|
+
return header.join(", "), csv_list
|
122
|
+
end
|
123
|
+
|
124
|
+
def revert_a_parameter_set_in_trashbox( id)
|
125
|
+
@param_records.transaction {
|
126
|
+
flag = @param_records.update_to_state_active(id)
|
127
|
+
return nil unless flag
|
128
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
129
|
+
cmd = "tar xvjf #{r_path}.tar.bz2"
|
130
|
+
system(cmd)
|
131
|
+
raise "Command #{cmd} failed" unless $? == 0
|
132
|
+
FileUtils.rm(r_path+".tar.bz2")
|
133
|
+
}
|
134
|
+
dump_in_yaml
|
135
|
+
return true
|
136
|
+
end
|
137
|
+
|
138
|
+
def delete_a_parameter_set( id)
|
139
|
+
@param_records.transaction {
|
140
|
+
flag = @param_records.destroy(id)
|
141
|
+
return nil unless flag
|
142
|
+
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
143
|
+
FileUtils.rm("#{r_path}.tar.bz2")
|
144
|
+
}
|
145
|
+
dump_in_yaml
|
146
|
+
return true
|
147
|
+
end
|
148
|
+
|
149
|
+
def dump_in_yaml
|
150
|
+
File.open( @db_folders.path_to_parameters_yaml( @sim_class), 'w') do |io|
|
151
|
+
YAML.dump( @param_records.list_all, io)
|
152
|
+
io.flush
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
end
|