nera 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +6 -0
- data/lib/nera/nera_database.rb +26 -2
- data/lib/nera/nera_db_folders.rb +13 -0
- data/lib/nera/nera_job_layer_controller.rb +30 -26
- data/lib/nera/nera_job_records.rb +5 -0
- data/lib/nera/nera_parameter_layer_controller.rb +20 -15
- data/lib/nera/nera_parameter_records.rb +18 -6
- data/lib/nera/nera_remote_connector.rb +50 -20
- data/lib/nera/nera_run_layer_controller.rb +26 -16
- data/lib/nera/nera_run_records.rb +11 -2
- data/lib/nera/nera_simulator_layer_controller.rb +4 -10
- data/lib/nera/nera_simulator_records.rb +8 -0
- data/lib/nera.rb +1 -1
- data/test/test_nera_database.rb +63 -1
- data/test/test_nera_db_folders.rb +8 -0
- data/test/test_nera_job_layer_controller.rb +22 -0
- data/test/test_nera_job_records.rb +13 -0
- data/test/test_nera_parameter_layer_controller.rb +16 -0
- data/test/test_nera_parameter_records.rb +69 -7
- data/test/test_nera_remote_connector.rb +25 -0
- data/test/test_nera_run_layer_controller.rb +27 -0
- data/test/test_nera_run_records.rb +56 -2
- data/test/test_nera_simulator_records.rb +29 -0
- metadata +2 -2
data/History.txt
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
== 0.2.0
|
2
|
+
* 1 major enhancement:
|
3
|
+
* logger is implemented. Log informatios are stored in "db_name"/Tables/logfile.txt.
|
4
|
+
* 2 bug fixes
|
5
|
+
* updated_at records of tables have been correctly implemented.
|
6
|
+
|
1
7
|
== 0.1.2
|
2
8
|
* 1 major enhancement:
|
3
9
|
* constructor of ParamLayerController now accepts String type simulator name.
|
data/lib/nera/nera_database.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'yaml'
|
3
|
-
#require 'filelock'
|
4
3
|
require 'pstore'
|
4
|
+
require 'observer'
|
5
5
|
require 'pp'
|
6
6
|
|
7
7
|
#* list of public methods
|
@@ -73,15 +73,24 @@ module NERA
|
|
73
73
|
# classes of values
|
74
74
|
VALUE_CLASSES = [ String, Symbol, Integer, Float, Date, DateTime, TrueClass, FalseClass, NilClass ]
|
75
75
|
|
76
|
+
# notify to updates to observer
|
77
|
+
include Observable
|
78
|
+
|
76
79
|
public
|
77
80
|
# ----------------------------------
|
78
|
-
def initialize( path_to_file )
|
81
|
+
def initialize( path_to_file, notify_observer_arg = nil )
|
79
82
|
@filename = path_to_file
|
80
83
|
@in_transaction = nil
|
84
|
+
@notify_observer_arg = notify_observer_arg
|
81
85
|
unless File.exist?(@filename)
|
82
86
|
raise "No such database : #{@filename}"
|
83
87
|
end
|
84
88
|
end
|
89
|
+
|
90
|
+
# ------------------------------------
|
91
|
+
def set_yaml_file( yml_path)
|
92
|
+
@yml_path = yml_path.to_s
|
93
|
+
end
|
85
94
|
|
86
95
|
# ------------------------------------
|
87
96
|
def transaction
|
@@ -94,6 +103,18 @@ module NERA
|
|
94
103
|
begin
|
95
104
|
@in_transaction = true
|
96
105
|
yield
|
106
|
+
if @notify_observer_arg
|
107
|
+
notify_observers( @notify_observer_arg)
|
108
|
+
else
|
109
|
+
notify_observers
|
110
|
+
end
|
111
|
+
if @yml_path
|
112
|
+
File.open( @yml_path,'w') do |io|
|
113
|
+
l = find_all do |r| true end
|
114
|
+
YAML.dump( l, io)
|
115
|
+
io.flush
|
116
|
+
end
|
117
|
+
end
|
97
118
|
ensure
|
98
119
|
@in_transaction = false
|
99
120
|
nil
|
@@ -140,6 +161,7 @@ module NERA
|
|
140
161
|
new_rec[:id] = new_id
|
141
162
|
@pstore[:max_id] = new_id
|
142
163
|
@pstore[new_id] = new_rec
|
164
|
+
changed
|
143
165
|
end
|
144
166
|
return new_id
|
145
167
|
end
|
@@ -235,6 +257,7 @@ module NERA
|
|
235
257
|
end
|
236
258
|
@pstore[id_to_update] = rec
|
237
259
|
flag = true
|
260
|
+
changed
|
238
261
|
end
|
239
262
|
return flag
|
240
263
|
end
|
@@ -267,6 +290,7 @@ module NERA
|
|
267
290
|
stat = nil
|
268
291
|
transaction do
|
269
292
|
stat = @pstore.delete(ids)
|
293
|
+
changed
|
270
294
|
end
|
271
295
|
return stat
|
272
296
|
else
|
data/lib/nera/nera_db_folders.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'nera_simulator_records'
|
3
3
|
require 'nera_job_records'
|
4
|
+
require 'logger'
|
4
5
|
|
5
6
|
|
6
7
|
module NERA
|
@@ -8,6 +9,14 @@ module NERA
|
|
8
9
|
# This class contains the information of the folder structure
|
9
10
|
class DbFolders
|
10
11
|
|
12
|
+
# -------------------------------------
|
13
|
+
# instance variables ------------------
|
14
|
+
# -------------------------------------
|
15
|
+
@db_folder # path to db_folder
|
16
|
+
@logger # logger instance
|
17
|
+
|
18
|
+
attr_reader :logger
|
19
|
+
|
11
20
|
#--------------------------------------
|
12
21
|
#-class methods -----------------------
|
13
22
|
#--------------------------------------
|
@@ -26,6 +35,10 @@ module NERA
|
|
26
35
|
Dir.glob( "#{@db_folder}/Simulator_classes/*.rb").each do |lib|
|
27
36
|
require lib
|
28
37
|
end
|
38
|
+
|
39
|
+
# logger instance
|
40
|
+
@logger = Logger.new( "#{@db_folder}/Tables/logfile.txt", 8)
|
41
|
+
@logger.level = Logger::INFO
|
29
42
|
end
|
30
43
|
|
31
44
|
def valid_db?( fold)
|
@@ -22,6 +22,7 @@ module NERA
|
|
22
22
|
|
23
23
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
24
24
|
@job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
25
|
+
@job_records.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
25
26
|
end
|
26
27
|
|
27
28
|
public
|
@@ -63,22 +64,29 @@ module NERA
|
|
63
64
|
d = @job_records.destroy( jid)
|
64
65
|
next unless d
|
65
66
|
sim_recs = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
66
|
-
|
67
|
+
sim_recs.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
68
|
+
found = sim_recs.list.find do |item| item[:name] == d[:simulator] end
|
69
|
+
sim_id = found[:id]
|
70
|
+
klass = NERA::Simulator.inherited_simulators.find do |sim|
|
71
|
+
sim.to_s == d[:simulator]
|
72
|
+
end
|
67
73
|
raise RuntimeError, "must not happen" unless klass
|
68
|
-
|
74
|
+
p_tab_path = @db_folders.path_to_parameters_table( klass)
|
75
|
+
param_recs = NERA::ParameterRecords.new( p_tab_path, sim_recs, sim_id)
|
76
|
+
param_recs.set_yaml_file( @db_folders.path_to_parameters_yaml( klass) )
|
77
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( klass, d[:parameter_id]), param_recs, d[:parameter_id] )
|
78
|
+
run_records.set_yaml_file( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]) )
|
69
79
|
run_records.transaction {
|
70
80
|
a = run_records.destroy_job_id( jid)
|
71
81
|
raise RuntimeError, "must not happen" unless a
|
72
82
|
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
73
83
|
}
|
74
|
-
File.open( @db_folders.path_to_runs_yaml( klass, d[:parameter_id]), 'w') do |io|
|
75
|
-
YAML.dump( run_records.list_all, io)
|
76
|
-
io.flush
|
77
|
-
end
|
78
84
|
destroyed_jobids << jid
|
85
|
+
@db_folders.logger.info(self.class.to_s) {
|
86
|
+
"canceled a job (jobid:#{jid})"
|
87
|
+
}
|
79
88
|
end
|
80
89
|
}
|
81
|
-
dump_in_yaml
|
82
90
|
destroyed_jobids = nil if destroyed_jobids.size == 0
|
83
91
|
return destroyed_jobids
|
84
92
|
end
|
@@ -96,19 +104,18 @@ module NERA
|
|
96
104
|
fpath = @db_folders.path_to_include_layer + filename
|
97
105
|
return nil unless File.exist?(fpath)
|
98
106
|
jinfo, rinfo = expand_and_parse( fpath)
|
99
|
-
run_layer_path, run_recs
|
107
|
+
run_layer_path, run_recs = check_consistency( jinfo, rinfo)
|
100
108
|
run_recs.transaction {
|
101
109
|
folder_path = fpath.sub(/.tar.bz2$/,'/')
|
102
110
|
file_move( folder_path, run_layer_path, jinfo[:run_ids])
|
103
111
|
update_tables( run_recs, jinfo, rinfo)
|
104
112
|
}
|
105
113
|
remove_files( fpath)
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
114
|
+
@db_folders.logger.info(self.class.to_s) {
|
115
|
+
"included a file (#{filename})"
|
116
|
+
}
|
117
|
+
|
110
118
|
}
|
111
|
-
dump_in_yaml
|
112
119
|
return true
|
113
120
|
end
|
114
121
|
|
@@ -139,12 +146,17 @@ module NERA
|
|
139
146
|
# check consistency of the simulator
|
140
147
|
conflicting unless jinfo[:simulator] == found[:simulator]
|
141
148
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
142
|
-
|
149
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
150
|
+
sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
151
|
+
sim.to_s == jinfo[:simulator]
|
152
|
+
end
|
143
153
|
conflicting unless jinfo[:simulator] == sim_class.to_s
|
144
154
|
|
145
155
|
# check consistency of the parameter
|
146
156
|
conflicting unless jinfo[:param_id] == found[:parameter_id]
|
147
|
-
|
157
|
+
f1 = sim_records.list.find do |item| item[:name] == jinfo[:simulator] end
|
158
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class) , sim_records, f1[:id] )
|
159
|
+
param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( sim_class) )
|
148
160
|
param = param_records.find_by_id( jinfo[:param_id])
|
149
161
|
jinfo[:parameters].each_pair do |key, value|
|
150
162
|
conflicting unless param[key] == value
|
@@ -153,8 +165,8 @@ module NERA
|
|
153
165
|
# check consistency of the runs
|
154
166
|
conflicting unless jinfo[:run_ids][0] == found[:run_id]
|
155
167
|
conflicting unless jinfo[:run_ids].size == found[:number_of_runs]
|
156
|
-
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]) )
|
157
|
-
|
168
|
+
run_recs = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, jinfo[:param_id]), param_records, jinfo[:param_id])
|
169
|
+
run_recs.set_yaml_file( @db_folders.path_to_runs_yaml( sim_class, jinfo[:param_id]) )
|
158
170
|
found_runs = run_recs.list_all_not_finished.find_all do |rec|
|
159
171
|
rec[:job_id] == jinfo[:job_id] and jinfo[:run_ids].include?(rec[:id])
|
160
172
|
end
|
@@ -164,7 +176,7 @@ module NERA
|
|
164
176
|
raise "Run info is not consistent with job info!" unless jinfo[:run_ids] == rinfo.keys.sort
|
165
177
|
|
166
178
|
run_layer_path = @db_folders.path_to_run_layer( sim_class, jinfo[:param_id])
|
167
|
-
return run_layer_path, run_recs
|
179
|
+
return run_layer_path, run_recs
|
168
180
|
end
|
169
181
|
|
170
182
|
def conflicting
|
@@ -223,16 +235,8 @@ module NERA
|
|
223
235
|
f = include(fname)
|
224
236
|
flag = nil unless f
|
225
237
|
end
|
226
|
-
dump_in_yaml
|
227
238
|
return flag
|
228
239
|
end
|
229
240
|
|
230
|
-
def dump_in_yaml
|
231
|
-
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
232
|
-
YAML.dump( @job_records.list_all, io)
|
233
|
-
io.flush
|
234
|
-
end
|
235
|
-
end
|
236
|
-
|
237
241
|
end
|
238
242
|
end
|
@@ -25,6 +25,11 @@ module NERA
|
|
25
25
|
@db = NERA::Database.new( db_file)
|
26
26
|
end
|
27
27
|
|
28
|
+
# argument is the path to jobs.yml
|
29
|
+
def set_yaml_file( yaml_path)
|
30
|
+
@db.set_yaml_file( yaml_path)
|
31
|
+
end
|
32
|
+
|
28
33
|
# if file already exists, returns false
|
29
34
|
def self.create_table( db_file)
|
30
35
|
NERA::Database.create_table( db_file)
|
@@ -20,17 +20,21 @@ module NERA
|
|
20
20
|
raise ArgumentError unless path_db_folder.is_a?(String)
|
21
21
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
22
22
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
23
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
24
|
+
|
23
25
|
found = sim_records.list.find do |rec|
|
24
26
|
rec[:name] == sim_name.to_s
|
25
27
|
end
|
26
28
|
raise "No such simulator." unless found
|
29
|
+
sim_id = found[:id]
|
27
30
|
@sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
28
31
|
sim.to_s == sim_name.to_s
|
29
32
|
end
|
30
33
|
raise "No such simulator." unless @sim_class
|
31
34
|
|
32
35
|
p_tab_path = @db_folders.path_to_parameters_table( @sim_class)
|
33
|
-
@param_records = NERA::ParameterRecords.new( p_tab_path,
|
36
|
+
@param_records = NERA::ParameterRecords.new( p_tab_path, sim_records, sim_id)
|
37
|
+
@param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( @sim_class) )
|
34
38
|
end
|
35
39
|
|
36
40
|
def parameters_list_in_csv
|
@@ -73,22 +77,24 @@ module NERA
|
|
73
77
|
pid = @param_records.add( p_hash)
|
74
78
|
return nil unless pid
|
75
79
|
b = @db_folders.create_new_parameter_folder( @sim_class, pid)
|
76
|
-
raise "Creation of a new folder for #{
|
80
|
+
raise "Creation of a new folder for #{@sim_calss}, #{pid} failed." unless b
|
77
81
|
path = @db_folders.path_to_runs_table( @sim_class, pid)
|
78
82
|
c = NERA::RunRecords.create_table( path)
|
79
83
|
raise "File #{path} already exists." unless c
|
84
|
+
@db_folders.logger.info(self.class.to_s) {
|
85
|
+
"created a new parameter set (#{@sim_class.to_s}/#{pid})"
|
86
|
+
}
|
80
87
|
}
|
81
|
-
dump_in_yaml
|
82
88
|
return pid
|
83
89
|
end
|
84
90
|
|
85
91
|
def move_a_parameter_set_into_trashbox( id)
|
86
92
|
return nil unless @param_records.find_by_id(id)
|
87
|
-
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id) )
|
93
|
+
run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( @sim_class, id), @param_records, id )
|
88
94
|
return nil if run_records.list_all_not_finished.size > 0
|
89
95
|
|
90
96
|
job_records = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
91
|
-
job_records.transaction {
|
97
|
+
job_records.transaction { # to lock the runs tables
|
92
98
|
@param_records.transaction {
|
93
99
|
flag = @param_records.update_to_state_trashbox(id)
|
94
100
|
return nil unless flag
|
@@ -97,9 +103,11 @@ module NERA
|
|
97
103
|
system(cmd)
|
98
104
|
raise "Command #{cmd} failed" unless $? == 0
|
99
105
|
FileUtils.rm_r(r_path)
|
106
|
+
@db_folders.logger.info(self.class.to_s) {
|
107
|
+
"moved a parameter set into the trashbox (#{@sim_class.to_s}/#{id})"
|
108
|
+
}
|
100
109
|
}
|
101
110
|
}
|
102
|
-
dump_in_yaml
|
103
111
|
return true
|
104
112
|
end
|
105
113
|
|
@@ -134,8 +142,10 @@ module NERA
|
|
134
142
|
system(cmd)
|
135
143
|
raise "Command #{cmd} failed" unless $? == 0
|
136
144
|
FileUtils.rm(r_path+".tar.bz2")
|
145
|
+
@db_folders.logger.info(self.class.to_s) {
|
146
|
+
"reverted a parameter set in the trashbox (#{@sim_class.to_s}/#{id})"
|
147
|
+
}
|
137
148
|
}
|
138
|
-
dump_in_yaml
|
139
149
|
return true
|
140
150
|
end
|
141
151
|
|
@@ -145,17 +155,12 @@ module NERA
|
|
145
155
|
return nil unless flag
|
146
156
|
r_path = @db_folders.path_to_run_layer( @sim_class, id).sub(/\/$/,'')
|
147
157
|
FileUtils.rm("#{r_path}.tar.bz2")
|
158
|
+
@db_folders.logger.info(self.class.to_s) {
|
159
|
+
"deleted a parameter set in the trashbox (#{@sim_class.to_s}/#{id})"
|
160
|
+
}
|
148
161
|
}
|
149
|
-
dump_in_yaml
|
150
162
|
return true
|
151
163
|
end
|
152
164
|
|
153
|
-
def dump_in_yaml
|
154
|
-
File.open( @db_folders.path_to_parameters_yaml( @sim_class), 'w') do |io|
|
155
|
-
YAML.dump( @param_records.list_all, io)
|
156
|
-
io.flush
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
165
|
end
|
161
166
|
end
|
@@ -21,13 +21,17 @@ module NERA
|
|
21
21
|
|
22
22
|
# simulator_specific keys
|
23
23
|
@sim_class
|
24
|
-
|
24
|
+
|
25
25
|
# argument is the path to database file
|
26
|
-
def initialize( db_file,
|
27
|
-
unless
|
28
|
-
raise ArgumentError, "
|
26
|
+
def initialize( db_file, sim_records, sim_id)
|
27
|
+
unless sim_records.is_a?( NERA::SimulatorRecords)
|
28
|
+
raise ArgumentError, "The second argument must be an instance of NERA::SimualtorRecords. #{sim_records.inspect}"
|
29
29
|
end
|
30
|
-
|
30
|
+
unless sim_id.is_a?(Integer)
|
31
|
+
raise ArgumentError, "sim_id must be an Integer. : #{sim_id.inspect}"
|
32
|
+
end
|
33
|
+
|
34
|
+
@sim_class = sim_records.get_class( sim_id)
|
31
35
|
@keys = ATTRIBUTES_PART.map do |k|
|
32
36
|
k[0]
|
33
37
|
end
|
@@ -41,7 +45,13 @@ module NERA
|
|
41
45
|
raise ArgumentError, "Invalid class. This class has a parameter : #{found}"
|
42
46
|
end
|
43
47
|
@keys += sim_keys
|
44
|
-
@db = NERA::Database.new( db_file)
|
48
|
+
@db = NERA::Database.new( db_file, sim_id)
|
49
|
+
@db.add_observer( sim_records)
|
50
|
+
end
|
51
|
+
|
52
|
+
# argument is the path to parameters.yml
|
53
|
+
def set_yaml_file( yaml_path)
|
54
|
+
@db.set_yaml_file( yaml_path)
|
45
55
|
end
|
46
56
|
|
47
57
|
# if file already exists, return false
|
@@ -169,6 +179,8 @@ module NERA
|
|
169
179
|
@db.update( found)
|
170
180
|
end
|
171
181
|
|
182
|
+
alias :update :touch
|
183
|
+
|
172
184
|
def destroy( id)
|
173
185
|
raise ArgumentError unless id.is_a?(Integer)
|
174
186
|
found = @db.find_by_id( id)
|
@@ -77,17 +77,21 @@ module NERA
|
|
77
77
|
Net::SSH.start( found[:host_url], found[:user], h) do |ssh|
|
78
78
|
str = ssh.exec!( cmd)
|
79
79
|
end
|
80
|
-
rescue Errno::ECONNREFUSED
|
80
|
+
rescue Errno::ECONNREFUSED => ex
|
81
81
|
$stderr.puts "failed"
|
82
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
82
83
|
return :connection_failed
|
83
|
-
rescue Errno::ENETUNREACH
|
84
|
+
rescue Errno::ENETUNREACH => ex
|
84
85
|
$stderr.puts "failed"
|
86
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
85
87
|
return :connection_failed
|
86
|
-
rescue
|
88
|
+
rescue SocketError => ex
|
87
89
|
$stderr.puts "failed"
|
90
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
88
91
|
return :connection_failed
|
89
|
-
rescue
|
92
|
+
rescue Net::SSH::Exception => ex
|
90
93
|
$stderr.puts "failed"
|
94
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
91
95
|
return :connection_failed
|
92
96
|
end
|
93
97
|
$stderr.puts "done"
|
@@ -131,20 +135,27 @@ module NERA
|
|
131
135
|
sftp.file.open(target_path, "w") do |f| f.puts contents end
|
132
136
|
ssh.exec!("chmod +x #{target_path}")
|
133
137
|
@job_records.update_to_state_copied( jid, host[:name])
|
138
|
+
@db_folders.logger.info(self.class.to_s) {
|
139
|
+
"transfered a file (#{File.basename(pjs)}) to #{host[:name]}"
|
140
|
+
}
|
134
141
|
end
|
135
142
|
end
|
136
143
|
end
|
137
|
-
rescue Errno::ECONNREFUSED
|
144
|
+
rescue Errno::ECONNREFUSED => ex
|
138
145
|
$stderr.puts "failed"
|
146
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
139
147
|
return :connection_failed
|
140
|
-
rescue Errno::ENETUNREACH
|
148
|
+
rescue Errno::ENETUNREACH => ex
|
141
149
|
$stderr.puts "failed"
|
150
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
142
151
|
return :connection_failed
|
143
|
-
rescue SocketError
|
152
|
+
rescue SocketError => ex
|
144
153
|
$stderr.puts "failed"
|
154
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
145
155
|
return :connection_failed
|
146
|
-
rescue Net::SSH::Exception
|
156
|
+
rescue Net::SSH::Exception => ex
|
147
157
|
$stderr.puts "failed"
|
158
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
148
159
|
return :connection_failed
|
149
160
|
end
|
150
161
|
$stderr.puts "done"
|
@@ -199,20 +210,27 @@ module NERA
|
|
199
210
|
@job_records.update_to_state_copied( jid, host[:name])
|
200
211
|
ssh.exec!("cd #{target_dir} && #{host[:submission_command]} #{target_path} &")
|
201
212
|
@job_records.update_to_state_submitted( jid, host[:name])
|
213
|
+
@db_folders.logger.info(self.class.to_s) {
|
214
|
+
"submitted a file (#{File.basename(pjs)}) to #{host[:name]}"
|
215
|
+
}
|
202
216
|
end
|
203
217
|
end
|
204
218
|
end
|
205
|
-
rescue Errno::ECONNREFUSED
|
219
|
+
rescue Errno::ECONNREFUSED => ex
|
206
220
|
$stderr.puts "failed"
|
221
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
207
222
|
return :connection_failed
|
208
|
-
rescue Errno::ENETUNREACH
|
223
|
+
rescue Errno::ENETUNREACH => ex
|
209
224
|
$stderr.puts "failed"
|
225
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
210
226
|
return :connection_failed
|
211
|
-
rescue SocketError
|
227
|
+
rescue SocketError => ex
|
212
228
|
$stderr.puts "failed"
|
229
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
213
230
|
return :connection_failed
|
214
|
-
rescue Net::SSH::Exception
|
231
|
+
rescue Net::SSH::Exception => ex
|
215
232
|
$stderr.puts "failed"
|
233
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
216
234
|
return :connection_failed
|
217
235
|
end
|
218
236
|
$stderr.puts "done"
|
@@ -235,17 +253,21 @@ module NERA
|
|
235
253
|
Net::SSH.start( host[:host_url], host[:user], h) do |ssh|
|
236
254
|
ls_str = ssh.exec!("ls #{host[:job_path]}")
|
237
255
|
end
|
238
|
-
rescue Errno::ECONNREFUSED
|
256
|
+
rescue Errno::ECONNREFUSED => ex
|
239
257
|
$stderr.puts "failed"
|
258
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
240
259
|
return :connection_failed
|
241
|
-
rescue Errno::ENETUNREACH
|
260
|
+
rescue Errno::ENETUNREACH => ex
|
242
261
|
$stderr.puts "failed"
|
262
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
243
263
|
return :connection_failed
|
244
|
-
rescue SocketError
|
264
|
+
rescue SocketError => ex
|
245
265
|
$stderr.puts "failed"
|
266
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
246
267
|
return :connection_failed
|
247
|
-
rescue Net::SSH::Exception
|
268
|
+
rescue Net::SSH::Exception => ex
|
248
269
|
$stderr.puts "failed"
|
270
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
249
271
|
return :connection_failed
|
250
272
|
end
|
251
273
|
$stderr.puts "done"
|
@@ -312,20 +334,28 @@ module NERA
|
|
312
334
|
$stderr.print " deleting #{sh_file}.....\t"
|
313
335
|
sftp.remove!(target_dir + '/' + sh_file)
|
314
336
|
$stderr.puts "done"
|
337
|
+
|
338
|
+
@db_folders.logger.info(self.class.to_s) {
|
339
|
+
"downloaded a data file (#{d_file}) from #{host[:name]}"
|
340
|
+
}
|
315
341
|
end
|
316
342
|
end
|
317
343
|
end
|
318
|
-
rescue Errno::ECONNREFUSED
|
344
|
+
rescue Errno::ECONNREFUSED => ex
|
319
345
|
$stderr.puts "failed"
|
346
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
320
347
|
return :connection_failed
|
321
|
-
rescue Errno::ENETUNREACH
|
348
|
+
rescue Errno::ENETUNREACH => ex
|
322
349
|
$stderr.puts "failed"
|
350
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
323
351
|
return :connection_failed
|
324
|
-
rescue SocketError
|
352
|
+
rescue SocketError => ex
|
325
353
|
$stderr.puts "failed"
|
354
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
326
355
|
return :connection_failed
|
327
|
-
rescue Net::SSH::Exception
|
356
|
+
rescue Net::SSH::Exception => ex
|
328
357
|
$stderr.puts "failed"
|
358
|
+
@db_folders.logger.error(self.class.to_s) { ex.to_s }
|
329
359
|
return :connection_failed
|
330
360
|
end
|
331
361
|
$stderr.puts "done"
|
@@ -25,10 +25,13 @@ module NERA
|
|
25
25
|
|
26
26
|
@db_folders = NERA::DbFolders.new( path_db_folder)
|
27
27
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
28
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
29
|
+
|
28
30
|
found = sim_records.list.find do |rec|
|
29
31
|
rec[:name] == sim_name.to_s
|
30
32
|
end
|
31
33
|
raise "No such simulator : #{sim_name.to_s}" unless found
|
34
|
+
sim_id = found[:id]
|
32
35
|
sim_class = NERA::Simulator.inherited_simulators.find do |sim|
|
33
36
|
sim.to_s == sim_name.to_s
|
34
37
|
end
|
@@ -36,7 +39,8 @@ module NERA
|
|
36
39
|
|
37
40
|
@sim_instance = sim_class.new
|
38
41
|
@param_id = param_id
|
39
|
-
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class),
|
42
|
+
param_records = NERA::ParameterRecords.new( @db_folders.path_to_parameters_table( sim_class), sim_records, sim_id)
|
43
|
+
param_records.set_yaml_file( @db_folders.path_to_parameters_yaml( sim_class) )
|
40
44
|
found = param_records.find_by_id( @param_id)
|
41
45
|
raise "No such parameter : #{param_id}." unless found
|
42
46
|
@sim_instance.class::Parameters.each do |pa|
|
@@ -48,7 +52,8 @@ module NERA
|
|
48
52
|
end
|
49
53
|
end
|
50
54
|
|
51
|
-
@run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, @param_id) )
|
55
|
+
@run_records = NERA::RunRecords.new( @db_folders.path_to_runs_table( sim_class, @param_id), param_records, @param_id )
|
56
|
+
@run_records.set_yaml_file( @db_folders.path_to_runs_yaml( @sim_instance.class, @param_id) )
|
52
57
|
end
|
53
58
|
|
54
59
|
def path_to_run_layer
|
@@ -80,6 +85,7 @@ module NERA
|
|
80
85
|
|
81
86
|
job_ids = []
|
82
87
|
job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
88
|
+
job_recs.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
83
89
|
js = NERA::JobScript.new( @db_folders)
|
84
90
|
job_recs.transaction {
|
85
91
|
@run_records.transaction {
|
@@ -89,13 +95,16 @@ module NERA
|
|
89
95
|
jid = job_recs.add( @sim_instance.class, @param_id, rid, num_runs_per_job)
|
90
96
|
@run_records.set_job_id( rid, num_runs_per_job, jid)
|
91
97
|
sim_records = NERA::SimulatorRecords.new( @db_folders.path_to_simulators_table)
|
98
|
+
sim_records.set_yaml_file( @db_folders.path_to_simulators_yaml )
|
92
99
|
found = sim_records.list.find do |rec| rec[:name] == @sim_instance.class.to_s end
|
93
100
|
path_script = js.create_script( jid, found[:id], @param_id, @sim_instance, run_stat)
|
94
101
|
job_ids << jid
|
102
|
+
@db_folders.logger.info(self.class.to_s) {
|
103
|
+
"created a job (#{@sim_instance.class.to_s}/#{@param_id}/#{rid}, #{num_runs_per_job} runs, jobid:#{jid})"
|
104
|
+
}
|
95
105
|
end
|
96
106
|
}
|
97
107
|
}
|
98
|
-
dump_in_yaml( job_recs)
|
99
108
|
return job_ids
|
100
109
|
end
|
101
110
|
|
@@ -132,6 +141,7 @@ module NERA
|
|
132
141
|
end
|
133
142
|
|
134
143
|
job_recs = NERA::JobRecords.new( @db_folders.path_to_jobs_table)
|
144
|
+
job_recs.set_yaml_file( @db_folders.path_to_jobs_yaml)
|
135
145
|
destroyed_jobids = []
|
136
146
|
job_recs.transaction {
|
137
147
|
@run_records.transaction {
|
@@ -142,10 +152,12 @@ module NERA
|
|
142
152
|
raise "must not happen" unless a
|
143
153
|
FileUtils.rm( @db_folders.path_to_job_script(jid) )
|
144
154
|
destroyed_jobids << jid
|
155
|
+
@db_folders.logger.info(self.class.to_s) {
|
156
|
+
"canceled a job (jobid:#{jid})"
|
157
|
+
}
|
145
158
|
end
|
146
159
|
}
|
147
160
|
}
|
148
|
-
dump_in_yaml( job_recs)
|
149
161
|
destroyed_jobids = nil if destroyed_jobids.size == 0
|
150
162
|
return destroyed_jobids
|
151
163
|
end
|
@@ -172,6 +184,10 @@ module NERA
|
|
172
184
|
FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
|
173
185
|
@sim_instance.__send__( method_str)
|
174
186
|
}
|
187
|
+
@db_folders.logger.info(self.class.to_s) {
|
188
|
+
"#{method_str} (#{@sim_instance.class.to_s}/#{@param_id})"
|
189
|
+
}
|
190
|
+
|
175
191
|
end
|
176
192
|
|
177
193
|
def analyze_all
|
@@ -180,24 +196,18 @@ module NERA
|
|
180
196
|
end
|
181
197
|
return nil unless found
|
182
198
|
|
199
|
+
found.each do |meth|
|
200
|
+
analyze( meth)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
=begin
|
183
204
|
FileUtils.cd( @db_folders.path_to_run_layer(@sim_instance.class, @param_id) ) {
|
184
205
|
found.each do |meth|
|
185
206
|
@sim_instance.__send__( meth)
|
186
207
|
end
|
187
208
|
}
|
188
209
|
end
|
189
|
-
|
190
|
-
private
|
191
|
-
def dump_in_yaml( job_recs)
|
192
|
-
File.open( @db_folders.path_to_runs_yaml( @sim_instance.class, @param_id), 'w') do |io|
|
193
|
-
YAML.dump( @run_records.list_all, io)
|
194
|
-
io.flush
|
195
|
-
end
|
196
|
-
File.open( @db_folders.path_to_jobs_yaml, 'w') do |io|
|
197
|
-
YAML.dump( job_recs.list_all, io)
|
198
|
-
io.flush
|
199
|
-
end
|
200
|
-
end
|
210
|
+
=end
|
201
211
|
|
202
212
|
end
|
203
213
|
end
|